Package org.apache.nifi.processors.standard

Class DeduplicateRecord

java.lang.Object
org.apache.nifi.components.AbstractConfigurableComponent
org.apache.nifi.processor.AbstractSessionFactoryProcessor
org.apache.nifi.processor.AbstractProcessor
org.apache.nifi.processors.standard.DeduplicateRecord
All Implemented Interfaces:
org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor
@SupportsBatching @InputRequirement(INPUT_REQUIRED) @SystemResourceConsideration(resource=MEMORY,description="The HashSet filter type will grow memory space proportionate to the number of unique records processed. The BloomFilter type will use constant memory regardless of the number of records processed.") @SystemResourceConsideration(resource=CPU,description="If a more advanced hash algorithm is chosen, the amount of time required to hash any particular record could increase substantially.") @Tags({"text","record","update","change","replace","modify","distinct","unique","filter","hash","dupe","duplicate","dedupe"}) @CapabilityDescription("This processor de-duplicates individual records within a record set. It can operate on a per-file basis using an in-memory hashset or bloom filter. When configured with a distributed map cache, it de-duplicates records across multiple files.") @WritesAttribute(attribute="record.count", description="Number of records written to the destination FlowFile.") @DynamicProperty(name="Name of the property.", value="A valid RecordPath to the record field to be included in the cache key used for deduplication.", description="A record\'s cache key is generated by combining the name of each dynamic property with its evaluated record value (as specified by the corresponding RecordPath).") @SeeAlso(classNames={"org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService","org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer","org.apache.nifi.processors.standard.DetectDuplicate"}) public class DeduplicateRecord extends org.apache.nifi.processor.AbstractProcessor

  • Field Details

    • JOIN_CHAR

      public static final char JOIN_CHAR
      See Also:

    • RECORD_COUNT_ATTRIBUTE

      public static final String RECORD_COUNT_ATTRIBUTE
      See Also:

    • RECORD_HASH_VALUE_ATTRIBUTE

      public static final String RECORD_HASH_VALUE_ATTRIBUTE
      See Also:

    • recordPathCache

      private volatile RecordPathCache recordPathCache

    • dynamicProperties

      private volatile List<org.apache.nifi.components.PropertyDescriptor> dynamicProperties

    • NONE_ALGORITHM_VALUE

      static final org.apache.nifi.components.AllowableValue NONE_ALGORITHM_VALUE

    • SHA256_ALGORITHM_VALUE

      static final org.apache.nifi.components.AllowableValue SHA256_ALGORITHM_VALUE

    • SHA512_ALGORITHM_VALUE

      static final org.apache.nifi.components.AllowableValue SHA512_ALGORITHM_VALUE

    • HASH_SET_VALUE

      static final org.apache.nifi.components.AllowableValue HASH_SET_VALUE

    • BLOOM_FILTER_VALUE

      static final org.apache.nifi.components.AllowableValue BLOOM_FILTER_VALUE

    • RECORD_READER

      static final org.apache.nifi.components.PropertyDescriptor RECORD_READER

    • RECORD_WRITER

      static final org.apache.nifi.components.PropertyDescriptor RECORD_WRITER

    • OPTION_SINGLE_FILE

      static final org.apache.nifi.components.AllowableValue OPTION_SINGLE_FILE

    • OPTION_MULTIPLE_FILES

      static final org.apache.nifi.components.AllowableValue OPTION_MULTIPLE_FILES

    • DEDUPLICATION_STRATEGY

      static final org.apache.nifi.components.PropertyDescriptor DEDUPLICATION_STRATEGY

    • DISTRIBUTED_MAP_CACHE

      static final org.apache.nifi.components.PropertyDescriptor DISTRIBUTED_MAP_CACHE

    • CACHE_IDENTIFIER

      static final org.apache.nifi.components.PropertyDescriptor CACHE_IDENTIFIER

    • PUT_CACHE_IDENTIFIER

      static final org.apache.nifi.components.PropertyDescriptor PUT_CACHE_IDENTIFIER

    • INCLUDE_ZERO_RECORD_FLOWFILES

      static final org.apache.nifi.components.PropertyDescriptor INCLUDE_ZERO_RECORD_FLOWFILES

    • RECORD_HASHING_ALGORITHM

      static final org.apache.nifi.components.PropertyDescriptor RECORD_HASHING_ALGORITHM

    • FILTER_TYPE

      static final org.apache.nifi.components.PropertyDescriptor FILTER_TYPE

    • FILTER_CAPACITY_HINT

      static final org.apache.nifi.components.PropertyDescriptor FILTER_CAPACITY_HINT

    • BLOOM_FILTER_FPP

      static final org.apache.nifi.components.PropertyDescriptor BLOOM_FILTER_FPP

    • PROPERTY_DESCRIPTORS

      private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTY_DESCRIPTORS

    • REL_DUPLICATE

      static final org.apache.nifi.processor.Relationship REL_DUPLICATE

    • REL_NON_DUPLICATE

      static final org.apache.nifi.processor.Relationship REL_NON_DUPLICATE

    • REL_ORIGINAL

      static final org.apache.nifi.processor.Relationship REL_ORIGINAL

    • REL_FAILURE

      static final org.apache.nifi.processor.Relationship REL_FAILURE

    • RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RELATIONSHIPS

    • mapCacheClient

      private DistributedMapCacheClient mapCacheClient

    • readerFactory

      private RecordReaderFactory readerFactory

    • writerFactory

      private RecordSetWriterFactory writerFactory

    • useInMemoryStrategy

      private boolean useInMemoryStrategy

    • STRING_SERIALIZER

      private static final Serializer<String> STRING_SERIALIZER

  • Constructor Details

    • DeduplicateRecord

      public DeduplicateRecord()

  • Method Details

    • getRelationships

      public Set<org.apache.nifi.processor.Relationship> getRelationships()
      Specified by:
      getRelationships in interface org.apache.nifi.processor.Processor
      Overrides:
      getRelationships in class org.apache.nifi.processor.AbstractSessionFactoryProcessor

    • getSupportedPropertyDescriptors

      public final List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()
      Overrides:
      getSupportedPropertyDescriptors in class org.apache.nifi.components.AbstractConfigurableComponent

    • getSupportedDynamicPropertyDescriptor

      protected org.apache.nifi.components.PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)
      Overrides:
      getSupportedDynamicPropertyDescriptor in class org.apache.nifi.components.AbstractConfigurableComponent

    • customValidate

      protected Collection<org.apache.nifi.components.ValidationResult> customValidate(org.apache.nifi.components.ValidationContext context)
      Overrides:
      customValidate in class org.apache.nifi.components.AbstractConfigurableComponent

    • onScheduled

      @OnScheduled public void onScheduled(org.apache.nifi.processor.ProcessContext context)

    • getFilter

      private DeduplicateRecord.FilterWrapper getFilter(org.apache.nifi.processor.ProcessContext context)

    • onTrigger

      public void onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session) throws org.apache.nifi.processor.exception.ProcessException
      Specified by:
      onTrigger in class org.apache.nifi.processor.AbstractProcessor
      Throws:
      org.apache.nifi.processor.exception.ProcessException

    • sendOrRemove

      private void sendOrRemove(org.apache.nifi.processor.ProcessSession session, org.apache.nifi.flowfile.FlowFile outputFlowFile, org.apache.nifi.processor.Relationship targetRelationship, String mimeType, boolean includeZeroRecordFlowFiles, WriteResult writeResult)

    • evaluateKeyFromDynamicProperties

      private String evaluateKeyFromDynamicProperties(org.apache.nifi.processor.ProcessContext context, Record record, org.apache.nifi.flowfile.FlowFile flowFile)