Package com.pulumi.aws.glue

Class Crawler

  • public class Crawler
extends com.pulumi.resources.CustomResource
    Manages a Glue Crawler. More information can be found in the [AWS Glue Developer Guide](https://docs.aws.amazon.com/glue/latest/dg/add-crawler.html) ## Example Usage ### DynamoDB Target Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerDynamodbTargetArgs;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var example = new Crawler("example", CrawlerArgs.builder()
             .dynamodbTargets(CrawlerDynamodbTargetArgs.builder()
                 .path("table-name")
                 .build())
             .databaseName(exampleAwsGlueCatalogDatabase.name())
             .name("example")
             .role(exampleAwsIamRole.arn())
             .build());
 
     }
 }
    ### JDBC Target Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerJdbcTargetArgs;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var example = new Crawler("example", CrawlerArgs.builder()
             .jdbcTargets(CrawlerJdbcTargetArgs.builder()
                 .connectionName(exampleAwsGlueConnection.name())
                 .path("database-name/%")
                 .build())
             .databaseName(exampleAwsGlueCatalogDatabase.name())
             .name("example")
             .role(exampleAwsIamRole.arn())
             .build());
 
     }
 }
    ### S3 Target Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var example = new Crawler("example", CrawlerArgs.builder()
             .s3Targets(CrawlerS3TargetArgs.builder()
                 .path(String.format("s3://%s", exampleAwsS3Bucket.bucket()))
                 .build())
             .databaseName(exampleAwsGlueCatalogDatabase.name())
             .name("example")
             .role(exampleAwsIamRole.arn())
             .build());
 
     }
 }
    ### Catalog Target Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerSchemaChangePolicyArgs;
 import com.pulumi.aws.glue.inputs.CrawlerCatalogTargetArgs;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var example = new Crawler("example", CrawlerArgs.builder()
             .schemaChangePolicy(CrawlerSchemaChangePolicyArgs.builder()
                 .deleteBehavior("LOG")
                 .build())
             .catalogTargets(CrawlerCatalogTargetArgs.builder()
                 .databaseName(exampleAwsGlueCatalogDatabase.name())
                 .tables(exampleAwsGlueCatalogTable.name())
                 .build())
             .databaseName(exampleAwsGlueCatalogDatabase.name())
             .name("example")
             .role(exampleAwsIamRole.arn())
             .configuration("""
 {
   \"Version\":1.0,
   \"Grouping\": {
     \"TableGroupingPolicy\": \"CombineCompatibleSchemas\"
   }
 }
             """)
             .build());
 
     }
 }
    ### MongoDB Target Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerMongodbTargetArgs;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var example = new Crawler("example", CrawlerArgs.builder()
             .mongodbTargets(CrawlerMongodbTargetArgs.builder()
                 .connectionName(exampleAwsGlueConnection.name())
                 .path("database-name/%")
                 .build())
             .databaseName(exampleAwsGlueCatalogDatabase.name())
             .name("example")
             .role(exampleAwsIamRole.arn())
             .build());
 
     }
 }
    ### Configuration Settings Example 
     
 package generated_program;
 
 import com.pulumi.Context;
 import com.pulumi.Pulumi;
 import com.pulumi.core.Output;
 import com.pulumi.aws.glue.Crawler;
 import com.pulumi.aws.glue.CrawlerArgs;
 import com.pulumi.aws.glue.inputs.CrawlerS3TargetArgs;
 import static com.pulumi.codegen.internal.Serialization.*;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Map;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
 public class App {
     public static void main(String[] args) {
         Pulumi.run(App::stack);
     }
 
     public static void stack(Context ctx) {
         var eventsCrawler = new Crawler("eventsCrawler", CrawlerArgs.builder()
             .s3Targets(CrawlerS3TargetArgs.builder()
                 .path(String.format("s3://%s", dataLakeBucket.bucket()))
                 .build())
             .databaseName(glueDatabase.name())
             .schedule("cron(0 1 * * ? *)")
             .name(String.format("events_crawler_%s", environmentName))
             .role(glueRole.arn())
             .tags(tags)
             .configuration(serializeJson(
                 jsonObject(
                     jsonProperty("Grouping", jsonObject(
                         jsonProperty("TableGroupingPolicy", "CombineCompatibleSchemas")
                     )),
                     jsonProperty("CrawlerOutput", jsonObject(
                         jsonProperty("Partitions", jsonObject(
                             jsonProperty("AddOrUpdateBehavior", "InheritFromTable")
                         ))
                     )),
                     jsonProperty("Version", 1)
                 )))
             .build());
 
     }
 }
    ## Import Using `pulumi import`, import Glue Crawlers using `name`. For example: ```sh $ pulumi import aws:glue/crawler:Crawler MyJob MyJob ```

    • Nested Class Summary

      • Nested classes/interfaces inherited from class com.pulumi.resources.CustomResource

        com.pulumi.resources.CustomResource.CustomResourceInternal

      • Nested classes/interfaces inherited from class com.pulumi.resources.Resource

        com.pulumi.resources.Resource.LazyField<T extends java.lang.Object>, com.pulumi.resources.Resource.LazyFields, com.pulumi.resources.Resource.ResourceInternal

    • Field Summary

      • Fields inherited from class com.pulumi.resources.Resource

        childResources, remote

    • Constructor Summary

      Constructors 
      Constructor Description
      Crawler​(java.lang.String name)  
      Crawler​(java.lang.String name, CrawlerArgs args)  
      Crawler​(java.lang.String name, CrawlerArgs args, com.pulumi.resources.CustomResourceOptions options)  

    • Constructor Detail

      • Crawler

        public Crawler​(java.lang.String name)
        Parameters:
        name - The _unique_ name of the resulting resource.

      • Crawler

        public Crawler​(java.lang.String name,
               CrawlerArgs args)
        Parameters:
        name - The _unique_ name of the resulting resource.
        args - The arguments to use to populate this resource's properties.

      • Crawler

        public Crawler​(java.lang.String name,
               CrawlerArgs args,
               @Nullable
               com.pulumi.resources.CustomResourceOptions options)
        Parameters:
        name - The _unique_ name of the resulting resource.
        args - The arguments to use to populate this resource's properties.
        options - A bag of options that control this resource's behavior.

    • Method Detail

      • arn

        public com.pulumi.core.Output<java.lang.String> arn()
        Returns:
        ARN of the crawler

      • catalogTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerCatalogTarget>>> catalogTargets()
        Returns:
        List of nested AWS Glue Data Catalog target arguments. See `catalogTarget` below.

      • classifiers

        public com.pulumi.core.Output<java.util.Optional<java.util.List<java.lang.String>>> classifiers()
        Returns:
        List of custom classifiers. By default, all AWS classifiers are included in a crawl, but these custom classifiers always override the default classifiers for a given classification.

      • configuration

        public com.pulumi.core.Output<java.util.Optional<java.lang.String>> configuration()
        Returns:
        JSON string of configuration information. For more details see [Setting Crawler Configuration Options](https://docs.aws.amazon.com/glue/latest/dg/crawler-configuration.html).

      • databaseName

        public com.pulumi.core.Output<java.lang.String> databaseName()
        Returns:
        Glue database where results are written.

      • deltaTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerDeltaTarget>>> deltaTargets()
        Returns:
        List of nested Delta Lake target arguments. See `deltaTarget` below.

      • description

        public com.pulumi.core.Output<java.util.Optional<java.lang.String>> description()
        Returns:
        Description of the crawler.

      • dynamodbTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerDynamodbTarget>>> dynamodbTargets()
        Returns:
        List of nested DynamoDB target arguments. See `dynamodbTarget` below.

      • hudiTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerHudiTarget>>> hudiTargets()
        Returns:
        List of nested Hudi target arguments. See `hudiTarget` below.

      • icebergTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerIcebergTarget>>> icebergTargets()
        Returns:
        List of nested Iceberg target arguments. See `icebergTarget` below.

      • jdbcTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerJdbcTarget>>> jdbcTargets()
        Returns:
        List of nested JDBC target arguments. See `jdbcTarget` below.

      • lakeFormationConfiguration

        public com.pulumi.core.Output<java.util.Optional<CrawlerLakeFormationConfiguration>> lakeFormationConfiguration()
        Returns:
        Lake Formation configuration settings for the crawler. See `lakeFormationConfiguration` below.

      • lineageConfiguration

        public com.pulumi.core.Output<java.util.Optional<CrawlerLineageConfiguration>> lineageConfiguration()
        Returns:
        Data lineage configuration settings for the crawler. See `lineageConfiguration` below.

      • mongodbTargets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerMongodbTarget>>> mongodbTargets()
        Returns:
        List of nested MongoDB target arguments. See `mongodbTarget` below.

      • name

        public com.pulumi.core.Output<java.lang.String> name()
        Returns:
        Name of the crawler.

      • recrawlPolicy

        public com.pulumi.core.Output<java.util.Optional<CrawlerRecrawlPolicy>> recrawlPolicy()
        Returns:
        Policy that specifies whether to crawl the entire dataset again, or to crawl only folders that were added since the last crawler run. See `recrawlPolicy` below.

      • region

        public com.pulumi.core.Output<java.lang.String> region()
        Returns:
        Region where this resource will be [managed](https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints). Defaults to the Region set in the provider configuration.

      • role

        public com.pulumi.core.Output<java.lang.String> role()
        Returns:
        IAM role friendly name (including path without leading slash), or ARN of an IAM role, used by the crawler to access other resources.

      • s3Targets

        public com.pulumi.core.Output<java.util.Optional<java.util.List<CrawlerS3Target>>> s3Targets()
        Returns:
        List of nested Amazon S3 target arguments. See `s3Target` below.

      • schedule

        public com.pulumi.core.Output<java.util.Optional<java.lang.String>> schedule()
        Returns:
        Cron expression used to specify the schedule. For more information, see [Time-Based Schedules for Jobs and Crawlers](https://docs.aws.amazon.com/glue/latest/dg/monitor-data-warehouse-schedule.html). For example, to run something every day at 12:15 UTC, you would specify: `cron(15 12 * * ? *)`.

      • schemaChangePolicy

        public com.pulumi.core.Output<java.util.Optional<CrawlerSchemaChangePolicy>> schemaChangePolicy()
        Returns:
        Policy for the crawler's update and deletion behavior. See `schemaChangePolicy` below.

      • securityConfiguration

        public com.pulumi.core.Output<java.util.Optional<java.lang.String>> securityConfiguration()
        Returns:
        Name of Security Configuration to be used by the crawler.

      • tablePrefix

        public com.pulumi.core.Output<java.util.Optional<java.lang.String>> tablePrefix()
        Returns:
        Table prefix used for catalog tables that are created.

      • tags

        public com.pulumi.core.Output<java.util.Optional<java.util.Map<java.lang.String,​java.lang.String>>> tags()
        Returns:
        Key-value map of resource tags. .If configured with a provider `defaultTags` configuration block present, tags with matching keys will overwrite those defined at the provider-level. > **NOTE:** Must specify at least one of `dynamodbTarget`, `jdbcTarget`, `s3Target`, `mongodbTarget` or `catalogTarget`.

      • tagsAll

        public com.pulumi.core.Output<java.util.Map<java.lang.String,​java.lang.String>> tagsAll()
        Returns:
        Map of tags assigned to the resource, including those inherited from the provider `defaultTags` configuration block.

      • get

        public static Crawler get​(java.lang.String name,
                          com.pulumi.core.Output<java.lang.String> id,
                          @Nullable
                          CrawlerState state,
                          @Nullable
                          com.pulumi.resources.CustomResourceOptions options)
        Get an existing Host resource's state with the given name, ID, and optional extra properties used to qualify the lookup.
        Parameters:
        name - The _unique_ name of the resulting resource.
        id - The _unique_ provider ID of the resource to lookup.
        state -
        options - Optional settings to control the behavior of the CustomResource.