diff --git a/setup.py b/setup.py index a642ad45..3e87b192 100644 --- a/setup.py +++ b/setup.py @@ -2,27 +2,28 @@ from setuptools import setup -setup(name='tap-salesforce', - version='2.2.0', - description='Singer.io tap for extracting data from the Salesforce API', - author='Stitch', - url='https://singer.io', - classifiers=['Programming Language :: Python :: 3 :: Only'], - py_modules=['tap_salesforce'], - install_requires=[ - 'requests==2.32.5', - 'singer-python==6.3.0', - 'xmltodict==1.0.2', - ], - entry_points=''' +setup( + name="tap-salesforce", + version="2.2.0", + description="Singer.io tap for extracting data from the Salesforce API", + author="Stitch", + url="https://singer.io", + classifiers=["Programming Language :: Python :: 3 :: Only"], + py_modules=["tap_salesforce"], + install_requires=[ + "requests==2.32.5", + "singer-python==6.3.0", + "xmltodict==1.0.2", + ], + entry_points=""" [console_scripts] tap-salesforce=tap_salesforce:main - ''', - packages=['tap_salesforce', 'tap_salesforce.salesforce'], - package_data = { - 'tap_salesforce/schemas': [ - # add schema.json filenames here - ] - }, - include_package_data=True, + """, + packages=["tap_salesforce", "tap_salesforce.salesforce"], + package_data={ + "tap_salesforce/schemas": [ + # add schema.json filenames here + ] + }, + include_package_data=True, ) diff --git a/spikes/test_threads.py b/spikes/test_threads.py index fcd7ba16..3cac5349 100755 --- a/spikes/test_threads.py +++ b/spikes/test_threads.py @@ -2,11 +2,12 @@ import sys import threading -from time import sleep from datetime import datetime +from time import sleep TIMER_LENGTH = 1 + def login(): print(f"In login, time is {datetime.now()}, threads is {threading.active_count()}") main_thread = threading.main_thread() @@ -17,10 +18,11 @@ def login(): # print(f"In login, the main thread died, so time to go") # sys.exit(0) - myTimer = threading.Timer(TIMER_LENGTH, login ) + myTimer = threading.Timer(TIMER_LENGTH, login) myTimer.daemon = True myTimer.start() + def main(): print("In Main") login() @@ -28,6 +30,4 @@ def main(): sys.exit(0) - - main() diff --git a/tap_salesforce/__init__.py b/tap_salesforce/__init__.py index 21c9797c..5b7efa92 100644 --- a/tap_salesforce/__init__.py +++ b/tap_salesforce/__init__.py @@ -1,141 +1,164 @@ #!/usr/bin/env python3 import json import sys + import singer import singer.utils as singer_utils from singer import metadata, metrics + import tap_salesforce.salesforce -from tap_salesforce.sync import (sync_stream, resume_syncing_bulk_query, get_stream_version) from tap_salesforce.salesforce import Salesforce from tap_salesforce.salesforce.bulk import Bulk from tap_salesforce.salesforce.exceptions import ( - TapSalesforceException, TapSalesforceQuotaExceededException, TapSalesforceBulkAPIDisabledException) + TapSalesforceBulkAPIDisabledException, + TapSalesforceException, + TapSalesforceQuotaExceededException, +) +from tap_salesforce.sync import ( + get_stream_version, + resume_syncing_bulk_query, + sync_stream, +) LOGGER = singer.get_logger() -REQUIRED_CONFIG_KEYS = ['refresh_token', - 'client_id', - 'client_secret', - 'start_date', - 'api_type'] +REQUIRED_CONFIG_KEYS = [ + "refresh_token", + "client_id", + "client_secret", + "start_date", + "api_type", +] CONFIG = { - 'refresh_token': None, - 'client_id': None, - 'client_secret': None, - 'start_date': None + "refresh_token": None, + "client_id": None, + "client_secret": None, + "start_date": None, } FORCED_FULL_TABLE = { # Does not support ordering by CreatedDate - 'BackgroundOperationResult', - 'LoginEvent', - 'LightningUriEvent', - 'UriEvent', - 'LogoutEvent', - 'ReportEvent', - 'PermissionSetEventStore', - 'ListViewEvent', - 'IdentityProviderEventStore', - 'ApiEvent', - 'BulkApiResultEventStore', - 'IdentityVerificationEvent', - 'LoginAsEvent', - 'FileEventStore', - 'ExternalEncryptionRootKey', - 'ActivityFieldHistory', - 'PendingOrderSummary' + "BackgroundOperationResult", + "LoginEvent", + "LightningUriEvent", + "UriEvent", + "LogoutEvent", + "ReportEvent", + "PermissionSetEventStore", + "ListViewEvent", + "IdentityProviderEventStore", + "ApiEvent", + "BulkApiResultEventStore", + "IdentityVerificationEvent", + "LoginAsEvent", + "FileEventStore", + "ExternalEncryptionRootKey", + "ActivityFieldHistory", + "PendingOrderSummary", } + def get_replication_key(sobject_name, fields): if sobject_name in FORCED_FULL_TABLE: return None - fields_list = [f['name'] for f in fields] + fields_list = [f["name"] for f in fields] - if 'SystemModstamp' in fields_list: - return 'SystemModstamp' - elif 'LastModifiedDate' in fields_list: - return 'LastModifiedDate' - elif 'CreatedDate' in fields_list: - return 'CreatedDate' - elif 'LoginTime' in fields_list and sobject_name == 'LoginHistory': - return 'LoginTime' + if "SystemModstamp" in fields_list: + return "SystemModstamp" + elif "LastModifiedDate" in fields_list: + return "LastModifiedDate" + elif "CreatedDate" in fields_list: + return "CreatedDate" + elif "LoginTime" in fields_list and sobject_name == "LoginHistory": + return "LoginTime" return None + def stream_is_selected(mdata): - return mdata.get((), {}).get('selected', False) + return mdata.get((), {}).get("selected", False) + def build_state(raw_state, catalog): state = {} - for catalog_entry in catalog['streams']: - tap_stream_id = catalog_entry['tap_stream_id'] + for catalog_entry in catalog["streams"]: + tap_stream_id = catalog_entry["tap_stream_id"] if tap_stream_id in FORCED_FULL_TABLE: - for metadata_entry in catalog_entry['metadata']: - if metadata_entry['breadcrumb'] == []: - metadata_entry['metadata']['forced-replication-method'] = 'FULL_TABLE' - metadata_entry['metadata'].pop('replication-key', None) + for metadata_entry in catalog_entry["metadata"]: + if metadata_entry["breadcrumb"] == []: + metadata_entry["metadata"]["forced-replication-method"] = ( + "FULL_TABLE" + ) + metadata_entry["metadata"].pop("replication-key", None) LOGGER.info("Forcing FULL_TABLE replication for %s", tap_stream_id) break - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_method = catalog_metadata.get((), {}).get('replication-method') + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_method = catalog_metadata.get((), {}).get("replication-method") - version = singer.get_bookmark(raw_state, - tap_stream_id, - 'version') + version = singer.get_bookmark(raw_state, tap_stream_id, "version") # Preserve state that deals with resuming an incomplete bulk job - if singer.get_bookmark(raw_state, tap_stream_id, 'JobID'): - job_id = singer.get_bookmark(raw_state, tap_stream_id, 'JobID') - batches = singer.get_bookmark(raw_state, tap_stream_id, 'BatchIDs') - current_bookmark = singer.get_bookmark(raw_state, tap_stream_id, 'JobHighestBookmarkSeen') - state = singer.write_bookmark(state, tap_stream_id, 'JobID', job_id) - state = singer.write_bookmark(state, tap_stream_id, 'BatchIDs', batches) - state = singer.write_bookmark(state, tap_stream_id, 'JobHighestBookmarkSeen', current_bookmark) - - if replication_method == 'INCREMENTAL': - replication_key = catalog_metadata.get((), {}).get('replication-key') - replication_key_value = singer.get_bookmark(raw_state, - tap_stream_id, - replication_key) + if singer.get_bookmark(raw_state, tap_stream_id, "JobID"): + job_id = singer.get_bookmark(raw_state, tap_stream_id, "JobID") + batches = singer.get_bookmark(raw_state, tap_stream_id, "BatchIDs") + current_bookmark = singer.get_bookmark( + raw_state, tap_stream_id, "JobHighestBookmarkSeen" + ) + state = singer.write_bookmark(state, tap_stream_id, "JobID", job_id) + state = singer.write_bookmark(state, tap_stream_id, "BatchIDs", batches) + state = singer.write_bookmark( + state, tap_stream_id, "JobHighestBookmarkSeen", current_bookmark + ) + + if replication_method == "INCREMENTAL": + replication_key = catalog_metadata.get((), {}).get("replication-key") + replication_key_value = singer.get_bookmark( + raw_state, tap_stream_id, replication_key + ) if version is not None: - state = singer.write_bookmark( - state, tap_stream_id, 'version', version) + state = singer.write_bookmark(state, tap_stream_id, "version", version) if replication_key_value is not None: state = singer.write_bookmark( - state, tap_stream_id, replication_key, replication_key_value) - elif replication_method == 'FULL_TABLE' and version is None: - state = singer.write_bookmark(state, tap_stream_id, 'version', version) + state, tap_stream_id, replication_key, replication_key_value + ) + elif replication_method == "FULL_TABLE" and version is None: + state = singer.write_bookmark(state, tap_stream_id, "version", version) return state + # pylint: disable=undefined-variable def create_property_schema(field, mdata, expected_pk_field): - field_name = field['name'] + field_name = field["name"] if field_name == expected_pk_field: mdata = metadata.write( - mdata, ('properties', field_name), 'inclusion', 'automatic') + mdata, ("properties", field_name), "inclusion", "automatic" + ) else: mdata = metadata.write( - mdata, ('properties', field_name), 'inclusion', 'available') + mdata, ("properties", field_name), "inclusion", "available" + ) property_schema, mdata = salesforce.field_to_property_schema(field, mdata) return (property_schema, mdata) + PK_OVERRIDES = { "LightningUriEvent": "EventIdentifier", } + # pylint: disable=too-many-branches,too-many-statements def do_discover(sf): """Describes a Salesforce instance's objects and generates a JSON schema for each field.""" global_description = sf.describe() - objects_to_discover = {o['name'] for o in global_description['sobjects']} + objects_to_discover = {o["name"] for o in global_description["sobjects"]} sf_custom_setting_objects = [] object_to_tag_references = {} @@ -144,15 +167,17 @@ def do_discover(sf): entries = [] # Check if the user has BULK API enabled - if sf.api_type == 'BULK' and not Bulk(sf).has_permissions(): - raise TapSalesforceBulkAPIDisabledException('This client does not have Bulk API permissions, received "API_DISABLED_FOR_ORG" error code') + if sf.api_type == "BULK" and not Bulk(sf).has_permissions(): + raise TapSalesforceBulkAPIDisabledException( + 'This client does not have Bulk API permissions, received "API_DISABLED_FOR_ORG" error code' + ) for sobject_name in objects_to_discover: - # Skip blacklisted SF objects depending on the api_type in use # ChangeEvent objects are not queryable via Bulk or REST (undocumented) - if sobject_name in sf.get_blacklisted_objects() \ - or sobject_name.endswith("ChangeEvent"): + if sobject_name in sf.get_blacklisted_objects() or sobject_name.endswith( + "ChangeEvent" + ): continue sobject_description = sf.describe(sobject_name) @@ -163,14 +188,20 @@ def do_discover(sf): sf_custom_setting_objects.append(sobject_name) elif sobject_name.endswith("__Tag"): relationship_field = next( - (f for f in sobject_description["fields"] if f.get("relationshipName") == "Item"), - None) + ( + f + for f in sobject_description["fields"] + if f.get("relationshipName") == "Item" + ), + None, + ) if relationship_field: # Map {"Object":"Object__Tag"} - object_to_tag_references[relationship_field["referenceTo"] - [0]] = sobject_name + object_to_tag_references[relationship_field["referenceTo"][0]] = ( + sobject_name + ) - fields = sobject_description['fields'] + fields = sobject_description["fields"] replication_key = get_replication_key(sobject_name, fields) unsupported_fields = set() @@ -183,7 +214,7 @@ def do_discover(sf): # Loop over the object's fields for f in fields: - field_name = f['name'] + field_name = f["name"] if field_name == expected_pk_field: found_expected_pk_field = True @@ -191,113 +222,137 @@ def do_discover(sf): property_schema, mdata = create_property_schema(f, mdata, expected_pk_field) # Compound Address fields and geolocations cannot be queried by the Bulk API - if f['type'] in ("address", "location") and sf.api_type == tap_salesforce.salesforce.BULK_API_TYPE: + if ( + f["type"] in ("address", "location") + and sf.api_type == tap_salesforce.salesforce.BULK_API_TYPE + ): unsupported_fields.add( - (field_name, 'cannot query compound address fields or geolocations with bulk API')) + ( + field_name, + "cannot query compound address fields or geolocations with bulk API", + ) + ) # we haven't been able to observe any records with a json field, so we # are marking it as unavailable until we have an example to work with - if f['type'] == "json": + if f["type"] == "json": unsupported_fields.add( - (field_name, 'do not currently support json fields - please contact support')) + ( + field_name, + "do not currently support json fields - please contact support", + ) + ) # Blacklisted fields are dependent on the api_type being used field_pair = (sobject_name, field_name) if field_pair in sf.get_blacklisted_fields(): unsupported_fields.add( - (field_name, sf.get_blacklisted_fields()[field_pair])) + (field_name, sf.get_blacklisted_fields()[field_pair]) + ) - inclusion = metadata.get( - mdata, ('properties', field_name), 'inclusion') + inclusion = metadata.get(mdata, ("properties", field_name), "inclusion") - if sf.select_fields_by_default and inclusion != 'unsupported': + if sf.select_fields_by_default and inclusion != "unsupported": mdata = metadata.write( - mdata, ('properties', field_name), 'selected-by-default', True) + mdata, ("properties", field_name), "selected-by-default", True + ) properties[field_name] = property_schema if replication_key: mdata = metadata.write( - mdata, ('properties', replication_key), 'inclusion', 'automatic') + mdata, ("properties", replication_key), "inclusion", "automatic" + ) # There are cases where compound fields are referenced by the associated # subfields but are not actually present in the field list - field_name_set = {f['name'] for f in fields} - filtered_unsupported_fields = [f for f in unsupported_fields if f[0] in field_name_set] - missing_unsupported_field_names = [f[0] for f in unsupported_fields if f[0] not in field_name_set] + field_name_set = {f["name"] for f in fields} + filtered_unsupported_fields = [ + f for f in unsupported_fields if f[0] in field_name_set + ] + missing_unsupported_field_names = [ + f[0] for f in unsupported_fields if f[0] not in field_name_set + ] if missing_unsupported_field_names: - LOGGER.info("Ignoring the following unsupported fields for object %s as they are missing from the field list: %s", - sobject_name, - ', '.join(sorted(missing_unsupported_field_names))) + LOGGER.info( + "Ignoring the following unsupported fields for object %s as they are missing from the field list: %s", + sobject_name, + ", ".join(sorted(missing_unsupported_field_names)), + ) if filtered_unsupported_fields: - LOGGER.info("Not syncing the following unsupported fields for object %s: %s", - sobject_name, - ', '.join(sorted([k for k, _ in filtered_unsupported_fields]))) + LOGGER.info( + "Not syncing the following unsupported fields for object %s: %s", + sobject_name, + ", ".join(sorted([k for k, _ in filtered_unsupported_fields])), + ) # Salesforce Objects are skipped when they do not have an expected pk field if not found_expected_pk_field: LOGGER.info( "Skipping Salesforce Object %s, as it has no %s field", - sobject_name, expected_pk_field) + sobject_name, + expected_pk_field, + ) continue # Any property added to unsupported_fields has metadata generated and # removed for prop, description in filtered_unsupported_fields: - if metadata.get(mdata, ('properties', prop), - 'selected-by-default'): - metadata.delete( - mdata, ('properties', prop), 'selected-by-default') + if metadata.get(mdata, ("properties", prop), "selected-by-default"): + metadata.delete(mdata, ("properties", prop), "selected-by-default") mdata = metadata.write( - mdata, ('properties', prop), 'unsupported-description', description) + mdata, ("properties", prop), "unsupported-description", description + ) mdata = metadata.write( - mdata, ('properties', prop), 'inclusion', 'unsupported') + mdata, ("properties", prop), "inclusion", "unsupported" + ) if replication_key: mdata = metadata.write( - mdata, (), 'valid-replication-keys', [replication_key]) + mdata, (), "valid-replication-keys", [replication_key] + ) else: - mdata = metadata.write( - mdata, - (), - 'forced-replication-method', - 'FULL_TABLE') + mdata = metadata.write(mdata, (), "forced-replication-method", "FULL_TABLE") - mdata = metadata.write(mdata, (), 'table-key-properties', [expected_pk_field]) + mdata = metadata.write(mdata, (), "table-key-properties", [expected_pk_field]) schema = { - 'type': 'object', - 'additionalProperties': False, - 'properties': properties + "type": "object", + "additionalProperties": False, + "properties": properties, } entry = { - 'stream': sobject_name, - 'tap_stream_id': sobject_name, - 'schema': schema, - 'metadata': metadata.to_list(mdata) + "stream": sobject_name, + "tap_stream_id": sobject_name, + "schema": schema, + "metadata": metadata.to_list(mdata), } entries.append(entry) # For each custom setting field, remove its associated tag from entries # See Blacklisting.md for more information - unsupported_tag_objects = [object_to_tag_references[f] - for f in sf_custom_setting_objects if f in object_to_tag_references] + unsupported_tag_objects = [ + object_to_tag_references[f] + for f in sf_custom_setting_objects + if f in object_to_tag_references + ] if unsupported_tag_objects: - LOGGER.info( #pylint:disable=logging-not-lazy - "Skipping the following Tag objects, Tags on Custom Settings Salesforce objects " + - "are not supported by the Bulk API:") + LOGGER.info( # pylint:disable=logging-not-lazy + "Skipping the following Tag objects, Tags on Custom Settings Salesforce objects " + + "are not supported by the Bulk API:" + ) LOGGER.info(unsupported_tag_objects) - entries = [e for e in entries if e['stream'] - not in unsupported_tag_objects] + entries = [e for e in entries if e["stream"] not in unsupported_tag_objects] - result = {'streams': entries} + result = {"streams": entries} json.dump(result, sys.stdout, indent=4) + def do_sync(sf, catalog, state): starting_stream = state.get("current_stream") @@ -308,16 +363,17 @@ def do_sync(sf, catalog, state): for catalog_entry in catalog["streams"]: stream_version = get_stream_version(catalog_entry, state) - stream = catalog_entry['stream'] - stream_alias = catalog_entry.get('stream_alias') + stream = catalog_entry["stream"] + stream_alias = catalog_entry.get("stream_alias") stream_name = catalog_entry["tap_stream_id"] activate_version_message = singer.ActivateVersionMessage( - stream=(stream_alias or stream), version=stream_version) + stream=(stream_alias or stream), version=stream_version + ) - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_key = catalog_metadata.get((), {}).get('replication-key') + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_key = catalog_metadata.get((), {}).get("replication-key") - mdata = metadata.to_map(catalog_entry['metadata']) + mdata = metadata.to_map(catalog_entry["metadata"]) if not stream_is_selected(mdata): LOGGER.info("%s: Skipping - not selected", stream_name) @@ -339,52 +395,75 @@ def do_sync(sf, catalog, state): state["current_stream"] = stream_name singer.write_state(state) - key_properties = metadata.to_map(catalog_entry['metadata']).get((), {}).get('table-key-properties') + key_properties = ( + metadata.to_map(catalog_entry["metadata"]) + .get((), {}) + .get("table-key-properties") + ) singer.write_schema( stream, - catalog_entry['schema'], + catalog_entry["schema"], key_properties, replication_key, - stream_alias) + stream_alias, + ) - job_id = singer.get_bookmark(state, catalog_entry['tap_stream_id'], 'JobID') - batch_ids = singer.get_bookmark(state, catalog_entry['tap_stream_id'], 'BatchIDs') + job_id = singer.get_bookmark(state, catalog_entry["tap_stream_id"], "JobID") + batch_ids = singer.get_bookmark( + state, catalog_entry["tap_stream_id"], "BatchIDs" + ) # Checking whether job_id list is not empty and batches list is not empty - if job_id and batch_ids : + if job_id and batch_ids: with metrics.record_counter(stream) as counter: - LOGGER.info("Found JobID from previous Bulk Query. Resuming sync for job: %s", job_id) + LOGGER.info( + "Found JobID from previous Bulk Query. Resuming sync for job: %s", + job_id, + ) # Resuming a sync should clear out the remaining state once finished - counter = resume_syncing_bulk_query(sf, catalog_entry, job_id, state, counter) + counter = resume_syncing_bulk_query( + sf, catalog_entry, job_id, state, counter + ) LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter.value) # Remove Job info from state once we complete this resumed query. One of a few cases could have occurred: # 1. The job succeeded, in which case make JobHighestBookmarkSeen the new bookmark # 2. The job partially completed, in which case make JobHighestBookmarkSeen the new bookmark, or # existing bookmark if no bookmark exists for the Job. # 3. The job completely failed, in which case maintain the existing bookmark, or None if no bookmark - state.get('bookmarks', {}).get(catalog_entry['tap_stream_id'], {}).pop('JobID', None) - state.get('bookmarks', {}).get(catalog_entry['tap_stream_id'], {}).pop('BatchIDs', None) - bookmark = state.get('bookmarks', {}).get(catalog_entry['tap_stream_id'], {}) \ - .pop('JobHighestBookmarkSeen', None) - existing_bookmark = state.get('bookmarks', {}).get(catalog_entry['tap_stream_id'], {}) \ - .pop(replication_key, None) + state.get("bookmarks", {}).get(catalog_entry["tap_stream_id"], {}).pop( + "JobID", None + ) + state.get("bookmarks", {}).get(catalog_entry["tap_stream_id"], {}).pop( + "BatchIDs", None + ) + bookmark = ( + state.get("bookmarks", {}) + .get(catalog_entry["tap_stream_id"], {}) + .pop("JobHighestBookmarkSeen", None) + ) + existing_bookmark = ( + state.get("bookmarks", {}) + .get(catalog_entry["tap_stream_id"], {}) + .pop(replication_key, None) + ) state = singer.write_bookmark( state, - catalog_entry['tap_stream_id'], + catalog_entry["tap_stream_id"], replication_key, - bookmark or existing_bookmark) # If job is removed, reset to existing bookmark or None + bookmark or existing_bookmark, + ) # If job is removed, reset to existing bookmark or None singer.write_state(state) else: # Tables with a replication_key or an empty bookmark will emit an # activate_version at the beginning of their sync - bookmark_is_empty = state.get('bookmarks', {}).get( - catalog_entry['tap_stream_id']) is None + bookmark_is_empty = ( + state.get("bookmarks", {}).get(catalog_entry["tap_stream_id"]) is None + ) if replication_key or bookmark_is_empty: singer.write_message(activate_version_message) - state = singer.write_bookmark(state, - catalog_entry['tap_stream_id'], - 'version', - stream_version) + state = singer.write_bookmark( + state, catalog_entry["tap_stream_id"], "version", stream_version + ) counter = sync_stream(sf, catalog_entry, state) LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter.value) @@ -392,6 +471,7 @@ def do_sync(sf, catalog, state): singer.write_state(state) LOGGER.info("Finished sync") + def main_impl(): args = singer_utils.parse_args(REQUIRED_CONFIG_KEYS) CONFIG.update(args.config) @@ -399,20 +479,21 @@ def main_impl(): sf = None try: # get lookback window from config - lookback_window = CONFIG.get('lookback_window') + lookback_window = CONFIG.get("lookback_window") lookback_window = int(lookback_window) if lookback_window else None sf = Salesforce( - refresh_token=CONFIG['refresh_token'], - sf_client_id=CONFIG['client_id'], - sf_client_secret=CONFIG['client_secret'], - quota_percent_total=CONFIG.get('quota_percent_total'), - quota_percent_per_run=CONFIG.get('quota_percent_per_run'), - is_sandbox=CONFIG.get('is_sandbox'), - select_fields_by_default=CONFIG.get('select_fields_by_default'), - default_start_date=CONFIG.get('start_date'), - api_type=CONFIG.get('api_type'), - lookback_window=lookback_window) + refresh_token=CONFIG["refresh_token"], + sf_client_id=CONFIG["client_id"], + sf_client_secret=CONFIG["client_secret"], + quota_percent_total=CONFIG.get("quota_percent_total"), + quota_percent_per_run=CONFIG.get("quota_percent_per_run"), + is_sandbox=CONFIG.get("is_sandbox"), + select_fields_by_default=CONFIG.get("select_fields_by_default"), + default_start_date=CONFIG.get("start_date"), + api_type=CONFIG.get("api_type"), + lookback_window=lookback_window, + ) sf.login() if args.discover: @@ -426,11 +507,13 @@ def main_impl(): if sf.rest_requests_attempted > 0: LOGGER.debug( "This job used %s REST requests towards the Salesforce quota.", - sf.rest_requests_attempted) + sf.rest_requests_attempted, + ) if sf.jobs_completed > 0: LOGGER.debug( "Replication used %s Bulk API jobs towards the Salesforce quota.", - sf.jobs_completed) + sf.jobs_completed, + ) if sf.login_timer: sf.login_timer.cancel() diff --git a/tap_salesforce/salesforce/__init__.py b/tap_salesforce/salesforce/__init__.py index fb6d5948..fce6ed60 100644 --- a/tap_salesforce/salesforce/__init__.py +++ b/tap_salesforce/salesforce/__init__.py @@ -1,20 +1,21 @@ import datetime import re import threading -import time + import backoff import requests -from requests.exceptions import RequestException import singer import singer.utils as singer_utils +from requests.exceptions import RequestException from singer import metadata, metrics from tap_salesforce.salesforce.bulk import Bulk -from tap_salesforce.salesforce.rest import Rest, API_VERSION from tap_salesforce.salesforce.exceptions import ( + Client406Error, TapSalesforceException, TapSalesforceQuotaExceededException, - Client406Error) +) +from tap_salesforce.salesforce.rest import API_VERSION, Rest LOGGER = singer.get_logger() @@ -24,151 +25,159 @@ BULK_API_TYPE = "BULK" REST_API_TYPE = "REST" -STRING_TYPES = set([ - 'id', - 'string', - 'picklist', - 'textarea', - 'phone', - 'url', - 'reference', - 'multipicklist', - 'combobox', - 'encryptedstring', - 'email', - 'complexvalue', # TODO: Unverified - 'masterrecord', - 'datacategorygroupreference' -]) - -NUMBER_TYPES = set([ - 'double', - 'currency', - 'percent' -]) - -DATE_TYPES = set([ - 'datetime', - 'date' -]) - -BINARY_TYPES = set([ - 'base64', - 'byte' -]) - -LOOSE_TYPES = set([ - 'anyType', - - # A calculated field's type can be any of the supported - # formula data types (see https://developer.salesforce.com/docs/#i1435527) - 'calculated' -]) +STRING_TYPES = set( + [ + "id", + "string", + "picklist", + "textarea", + "phone", + "url", + "reference", + "multipicklist", + "combobox", + "encryptedstring", + "email", + "complexvalue", # TODO: Unverified + "masterrecord", + "datacategorygroupreference", + ] +) + +NUMBER_TYPES = set(["double", "currency", "percent"]) + +DATE_TYPES = set(["datetime", "date"]) + +BINARY_TYPES = set(["base64", "byte"]) + +LOOSE_TYPES = set( + [ + "anyType", + # A calculated field's type can be any of the supported + # formula data types (see https://developer.salesforce.com/docs/#i1435527) + "calculated", + ] +) # The following objects are not supported by the bulk API. -UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = set(['FieldSecurityClassification', - 'WorkStepStatus', - 'ShiftStatus', - 'WorkOrderStatus', - 'WorkOrderLineItemStatus', - 'ServiceAppointmentStatus', - 'SolutionStatus', - 'ContractStatus', - 'RecentlyViewed', - 'DeclinedEventRelation', - 'AcceptedEventRelation', - 'TaskStatus', - 'PartnerRole', - 'TaskPriority', - 'CaseStatus', - 'UndecidedEventRelation', - 'OrderStatus']) +UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = set( + [ + "FieldSecurityClassification", + "WorkStepStatus", + "ShiftStatus", + "WorkOrderStatus", + "WorkOrderLineItemStatus", + "ServiceAppointmentStatus", + "SolutionStatus", + "ContractStatus", + "RecentlyViewed", + "DeclinedEventRelation", + "AcceptedEventRelation", + "TaskStatus", + "PartnerRole", + "TaskPriority", + "CaseStatus", + "UndecidedEventRelation", + "OrderStatus", + ] +) # The following objects have certain WHERE clause restrictions so we exclude them. -QUERY_RESTRICTED_SALESFORCE_OBJECTS = set(['Announcement', - 'ContentDocumentLink', - 'CollaborationGroupRecord', - 'Vote', - 'IdeaComment', - 'FieldDefinition', - 'PlatformAction', - 'UserEntityAccess', - 'RelationshipInfo', - 'ContentFolderMember', - 'ContentFolderItem', - 'SearchLayout', - 'SiteDetail', - 'EntityParticle', - 'OwnerChangeOptionInfo', - 'DataStatistics', - 'UserFieldAccess', - 'PicklistValueInfo', - 'RelationshipDomain', - 'FlexQueueItem', - 'NetworkUserHistoryRecent', - 'FieldHistoryArchive', - 'RecordActionHistory', - 'FlowVersionView', - 'FlowVariableView', - 'AppTabMember', - 'ColorDefinition', - 'DatacloudDandBCompany', # Not filterable without a criteria. - 'DatacloudAddress', # Transient queries are not implemented - 'FlowTestView', # A filter on a reified column is required [FlowDefinitionViewId,DurableId] - 'RelatedListColumnDefinition', # A filter on a reified column is required [RelatedListDefinitionId,DurableId], - 'RelatedListDefinition', # A filter on a reified column is required [ParentEntityDefinitionId,DurableId], - 'ApexTypeImplementor', # A filter on a reified column is required [InterfaceName,DurableId] - 'IconDefinition',]) +QUERY_RESTRICTED_SALESFORCE_OBJECTS = set( + [ + "Announcement", + "ContentDocumentLink", + "CollaborationGroupRecord", + "Vote", + "IdeaComment", + "FieldDefinition", + "PlatformAction", + "UserEntityAccess", + "RelationshipInfo", + "ContentFolderMember", + "ContentFolderItem", + "SearchLayout", + "SiteDetail", + "EntityParticle", + "OwnerChangeOptionInfo", + "DataStatistics", + "UserFieldAccess", + "PicklistValueInfo", + "RelationshipDomain", + "FlexQueueItem", + "NetworkUserHistoryRecent", + "FieldHistoryArchive", + "RecordActionHistory", + "FlowVersionView", + "FlowVariableView", + "AppTabMember", + "ColorDefinition", + "DatacloudDandBCompany", # Not filterable without a criteria. + "DatacloudAddress", # Transient queries are not implemented + "FlowTestView", # A filter on a reified column is required [FlowDefinitionViewId,DurableId] + "RelatedListColumnDefinition", # A filter on a reified column is required [RelatedListDefinitionId,DurableId], + "RelatedListDefinition", # A filter on a reified column is required [ParentEntityDefinitionId,DurableId], + "ApexTypeImplementor", # A filter on a reified column is required [InterfaceName,DurableId] + "IconDefinition", + ] +) # The following objects are not supported by the query method being used. -QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS = set(['DataType', - 'ListViewChartInstance', - 'FeedLike', - 'OutgoingEmail', - 'OutgoingEmailRelation', - 'FeedSignal', - 'ActivityHistory', - 'EmailStatus', - 'UserRecordAccess', - 'Name', - 'AggregateResult', - 'OpenActivity', - 'ProcessInstanceHistory', - 'OwnedContentDocument', - 'FolderedContentDocument', - 'FeedTrackedChange', - 'CombinedAttachment', - 'AttachedContentDocument', - 'ContentBody', - 'NoteAndAttachment', - 'LookedUpFromActivity', - 'AttachedContentNote', - 'QuoteTemplateRichTextData']) +QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS = set( + [ + "DataType", + "ListViewChartInstance", + "FeedLike", + "OutgoingEmail", + "OutgoingEmailRelation", + "FeedSignal", + "ActivityHistory", + "EmailStatus", + "UserRecordAccess", + "Name", + "AggregateResult", + "OpenActivity", + "ProcessInstanceHistory", + "OwnedContentDocument", + "FolderedContentDocument", + "FeedTrackedChange", + "CombinedAttachment", + "AttachedContentDocument", + "ContentBody", + "NoteAndAttachment", + "LookedUpFromActivity", + "AttachedContentNote", + "QuoteTemplateRichTextData", + ] +) + def log_backoff_attempt(details): - LOGGER.info("ConnectionError detected, triggering backoff: %d try", details.get("tries")) + LOGGER.info( + "ConnectionError detected, triggering backoff: %d try", details.get("tries") + ) -def field_to_property_schema(field, mdata): # pylint:disable=too-many-branches +def field_to_property_schema(field, mdata): # pylint:disable=too-many-branches property_schema = {} - field_name = field['name'] - sf_type = field['type'] + field_name = field["name"] + sf_type = field["type"] if sf_type in STRING_TYPES: - property_schema['type'] = "string" + property_schema["type"] = "string" elif sf_type in DATE_TYPES: date_type = {"type": "string", "format": "date-time"} string_type = {"type": ["string", "null"]} property_schema["anyOf"] = [date_type, string_type] elif sf_type == "boolean": - property_schema['type'] = "boolean" + property_schema["type"] = "boolean" elif sf_type in NUMBER_TYPES: - property_schema['type'] = "number" + property_schema["type"] = "number" elif sf_type == "address": - property_schema['type'] = "object" - property_schema['properties'] = { + property_schema["type"] = "object" + property_schema["properties"] = { "street": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "postalCode": {"type": ["null", "string"]}, @@ -176,51 +185,57 @@ def field_to_property_schema(field, mdata): # pylint:disable=too-many-branches "country": {"type": ["null", "string"]}, "longitude": {"type": ["null", "number"]}, "latitude": {"type": ["null", "number"]}, - "geocodeAccuracy": {"type": ["null", "string"]} + "geocodeAccuracy": {"type": ["null", "string"]}, } elif sf_type in ("int", "long"): - property_schema['type'] = "integer" + property_schema["type"] = "integer" elif sf_type == "time": - property_schema['type'] = "string" + property_schema["type"] = "string" elif sf_type in LOOSE_TYPES: return property_schema, mdata # No type = all types elif sf_type in BINARY_TYPES: - mdata = metadata.write(mdata, ('properties', field_name), "inclusion", "unsupported") - mdata = metadata.write(mdata, ('properties', field_name), - "unsupported-description", "binary data") + mdata = metadata.write( + mdata, ("properties", field_name), "inclusion", "unsupported" + ) + mdata = metadata.write( + mdata, ("properties", field_name), "unsupported-description", "binary data" + ) return property_schema, mdata - elif sf_type == 'location': + elif sf_type == "location": # geo coordinates are numbers or objects divided into two fields for lat/long - property_schema['type'] = ["number", "object", "null"] - property_schema['properties'] = { + property_schema["type"] = ["number", "object", "null"] + property_schema["properties"] = { "longitude": {"type": ["null", "number"]}, - "latitude": {"type": ["null", "number"]} + "latitude": {"type": ["null", "number"]}, } - elif sf_type == 'json': - property_schema['type'] = "string" + elif sf_type == "json": + property_schema["type"] = "string" else: raise TapSalesforceException("Found unsupported type: {}".format(sf_type)) # The nillable field cannot be trusted - if field_name != 'Id' and sf_type != 'location' and sf_type not in DATE_TYPES: - property_schema['type'] = ["null", property_schema['type']] + if field_name != "Id" and sf_type != "location" and sf_type not in DATE_TYPES: + property_schema["type"] = ["null", property_schema["type"]] return property_schema, mdata -class Salesforce(): + +class Salesforce: # pylint: disable=too-many-instance-attributes,too-many-arguments,too-many-positional-arguments - def __init__(self, - refresh_token=None, - token=None, - sf_client_id=None, - sf_client_secret=None, - quota_percent_per_run=None, - quota_percent_total=None, - is_sandbox=None, - select_fields_by_default=None, - default_start_date=None, - api_type=None, - lookback_window=None): + def __init__( + self, + refresh_token=None, + token=None, + sf_client_id=None, + sf_client_secret=None, + quota_percent_per_run=None, + quota_percent_total=None, + is_sandbox=None, + select_fields_by_default=None, + default_start_date=None, + api_type=None, + lookback_window=None, + ): self.api_type = api_type.upper() if api_type else None self.refresh_token = refresh_token self.token = token @@ -229,16 +244,26 @@ def __init__(self, self.session = requests.Session() self.access_token = None self.instance_url = None - if isinstance(quota_percent_per_run, str) and quota_percent_per_run.strip() == '': + if ( + isinstance(quota_percent_per_run, str) + and quota_percent_per_run.strip() == "" + ): quota_percent_per_run = None - if isinstance(quota_percent_total, str) and quota_percent_total.strip() == '': + if isinstance(quota_percent_total, str) and quota_percent_total.strip() == "": quota_percent_total = None - self.quota_percent_per_run = float( - quota_percent_per_run) if quota_percent_per_run is not None else 25 - self.quota_percent_total = float( - quota_percent_total) if quota_percent_total is not None else 80 - self.is_sandbox = is_sandbox is True or (isinstance(is_sandbox, str) and is_sandbox.lower() == 'true') - self.select_fields_by_default = select_fields_by_default is True or (isinstance(select_fields_by_default, str) and select_fields_by_default.lower() == 'true') + self.quota_percent_per_run = ( + float(quota_percent_per_run) if quota_percent_per_run is not None else 25 + ) + self.quota_percent_total = ( + float(quota_percent_total) if quota_percent_total is not None else 80 + ) + self.is_sandbox = is_sandbox is True or ( + isinstance(is_sandbox, str) and is_sandbox.lower() == "true" + ) + self.select_fields_by_default = select_fields_by_default is True or ( + isinstance(select_fields_by_default, str) + and select_fields_by_default.lower() == "true" + ) self.default_start_date = default_start_date self.rest_requests_attempted = 0 self.jobs_completed = 0 @@ -255,7 +280,7 @@ def _get_standard_headers(self): # pylint: disable=anomalous-backslash-in-string,line-too-long def check_rest_quota_usage(self, headers): - match = re.search('^api-usage=(\d+)/(\d+)$', headers.get('Sforce-Limit-Info')) + match = re.search("^api-usage=(\d+)/(\d+)$", headers.get("Sforce-Limit-Info")) if match is None: return @@ -268,51 +293,76 @@ def check_rest_quota_usage(self, headers): max_requests_for_run = int((self.quota_percent_per_run * allotted) / 100) if percent_used_from_total > self.quota_percent_total: - total_message = ("Salesforce has reported {}/{} ({:3.2f}%) total REST quota " + - "used across all Salesforce Applications. Terminating " + - "replication to not continue past configured percentage " + - "of {}% total quota.").format(remaining, - allotted, - percent_used_from_total, - self.quota_percent_total) + total_message = ( + "Salesforce has reported {}/{} ({:3.2f}%) total REST quota " + + "used across all Salesforce Applications. Terminating " + + "replication to not continue past configured percentage " + + "of {}% total quota." + ).format( + remaining, allotted, percent_used_from_total, self.quota_percent_total + ) raise TapSalesforceQuotaExceededException(total_message) elif self.rest_requests_attempted > max_requests_for_run: - partial_message = ("This replication job has made {} REST requests ({:3.2f}% of " + - "total quota). Terminating replication due to allotted " + - "quota of {}% per replication.").format(self.rest_requests_attempted, - (self.rest_requests_attempted / allotted) * 100, - self.quota_percent_per_run) + partial_message = ( + "This replication job has made {} REST requests ({:3.2f}% of " + + "total quota). Terminating replication due to allotted " + + "quota of {}% per replication." + ).format( + self.rest_requests_attempted, + (self.rest_requests_attempted / allotted) * 100, + self.quota_percent_per_run, + ) raise TapSalesforceQuotaExceededException(partial_message) # pylint: disable=too-many-arguments,too-many-positional-arguments - @backoff.on_exception(backoff.expo, - (requests.exceptions.ConnectionError, requests.exceptions.Timeout, Client406Error), - max_tries=6, - factor=2, - on_backoff=log_backoff_attempt) - def _make_request(self, http_method, url, headers=None, body=None, stream=False, params=None): - request_timeout = 5 * 60 # 5 minute request timeout + @backoff.on_exception( + backoff.expo, + ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + Client406Error, + ), + max_tries=6, + factor=2, + on_backoff=log_backoff_attempt, + ) + def _make_request( + self, http_method, url, headers=None, body=None, stream=False, params=None + ): + request_timeout = 5 * 60 # 5 minute request timeout try: if http_method == "GET": - LOGGER.info("Making %s request to %s with params: %s", http_method, url, params) - resp = self.session.get(url, - headers=headers, - stream=stream, - params=params, - timeout=request_timeout,) + LOGGER.info( + "Making %s request to %s with params: %s", http_method, url, params + ) + resp = self.session.get( + url, + headers=headers, + stream=stream, + params=params, + timeout=request_timeout, + ) elif http_method == "POST": - LOGGER.info("Making %s request to %s with body %s", http_method, url, body) - resp = self.session.post(url, - headers=headers, - data=body, - timeout=request_timeout,) + LOGGER.info( + "Making %s request to %s with body %s", http_method, url, body + ) + resp = self.session.post( + url, + headers=headers, + data=body, + timeout=request_timeout, + ) else: raise TapSalesforceException("Unsupported HTTP method") except requests.exceptions.ConnectionError as connection_err: - LOGGER.error('Took longer than %s seconds to connect to the server', request_timeout) + LOGGER.error( + "Took longer than %s seconds to connect to the server", request_timeout + ) raise connection_err except requests.exceptions.Timeout as timeout_err: - LOGGER.error('Took longer than %s seconds to hear from the server', request_timeout) + LOGGER.error( + "Took longer than %s seconds to hear from the server", request_timeout + ) raise timeout_err if resp.status_code == 406: @@ -323,7 +373,7 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, except RequestException as ex: raise ex - if resp.headers.get('Sforce-Limit-Info') is not None: + if resp.headers.get("Sforce-Limit-Info") is not None: self.rest_requests_attempted += 1 self.check_rest_quota_usage(resp.headers) @@ -331,37 +381,52 @@ def _make_request(self, http_method, url, headers=None, body=None, stream=False, def login(self): if self.is_sandbox: - login_url = 'https://test.salesforce.com/services/oauth2/token' + login_url = "https://test.salesforce.com/services/oauth2/token" else: - login_url = 'https://login.salesforce.com/services/oauth2/token' + login_url = "https://login.salesforce.com/services/oauth2/token" - login_body = {'grant_type': 'refresh_token', 'client_id': self.sf_client_id, - 'client_secret': self.sf_client_secret, 'refresh_token': self.refresh_token} + login_body = { + "grant_type": "refresh_token", + "client_id": self.sf_client_id, + "client_secret": self.sf_client_secret, + "refresh_token": self.refresh_token, + } LOGGER.info("Attempting login via OAuth2") resp = None try: - resp = self._make_request("POST", login_url, body=login_body, headers={"Content-Type": "application/x-www-form-urlencoded"}) + resp = self._make_request( + "POST", + login_url, + body=login_body, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) LOGGER.info("OAuth2 login successful") auth = resp.json() - self.access_token = auth['access_token'] - self.instance_url = auth['instance_url'] + self.access_token = auth["access_token"] + self.instance_url = auth["instance_url"] except Exception as e: error_message = str(e) - if resp is None and hasattr(e, 'response') and e.response is not None: #pylint:disable=no-member - resp = e.response #pylint:disable=no-member + if resp is None and hasattr(e, "response") and e.response is not None: # pylint:disable=no-member + resp = e.response # pylint:disable=no-member # NB: requests.models.Response is always falsy here. It is false if status code >= 400 if isinstance(resp, requests.models.Response): - error_message = error_message + ", Response from Salesforce: {}".format(resp.text) + error_message = error_message + ", Response from Salesforce: {}".format( + resp.text + ) raise Exception(error_message) from e finally: LOGGER.info("Starting new login timer") - self.login_timer = threading.Timer(REFRESH_TOKEN_EXPIRATION_PERIOD, self.login) - self.login_timer.daemon = True # The timer should be a daemon thread so the process exits. + self.login_timer = threading.Timer( + REFRESH_TOKEN_EXPIRATION_PERIOD, self.login + ) + self.login_timer.daemon = ( + True # The timer should be a daemon thread so the process exits. + ) self.login_timer.start() def describe(self, sobject=None): @@ -377,51 +442,62 @@ def describe(self, sobject=None): url = self.data_url.format(self.instance_url, API_VERSION, endpoint) with metrics.http_request_timer("describe") as timer: - timer.tags['endpoint'] = endpoint_tag - resp = self._make_request('GET', url, headers=headers) + timer.tags["endpoint"] = endpoint_tag + resp = self._make_request("GET", url, headers=headers) return resp.json() def _get_selected_properties(self, catalog_entry): - mdata = metadata.to_map(catalog_entry['metadata']) - properties = catalog_entry['schema'].get('properties', {}) - - return [k for k in properties.keys() - if singer.should_sync_field(metadata.get(mdata, ('properties', k), 'inclusion'), - metadata.get(mdata, ('properties', k), 'selected'), - self.select_fields_by_default)] - + mdata = metadata.to_map(catalog_entry["metadata"]) + properties = catalog_entry["schema"].get("properties", {}) + + return [ + k + for k in properties.keys() + if singer.should_sync_field( + metadata.get(mdata, ("properties", k), "inclusion"), + metadata.get(mdata, ("properties", k), "selected"), + self.select_fields_by_default, + ) + ] def get_start_date(self, state, catalog_entry): """ - return start date if state is not provided - else return bookmark from the state by subtracting lookback if provided + return start date if state is not provided + else return bookmark from the state by subtracting lookback if provided """ - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_key = catalog_metadata.get((), {}).get('replication-key') + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_key = catalog_metadata.get((), {}).get("replication-key") # get bookmark value from the state - bookmark_value = singer.get_bookmark(state, catalog_entry['tap_stream_id'], replication_key) + bookmark_value = singer.get_bookmark( + state, catalog_entry["tap_stream_id"], replication_key + ) sync_start_date = bookmark_value or self.default_start_date # if the state contains a bookmark, subtract the lookback window from the bookmark if bookmark_value and self.lookback_window: - sync_start_date = singer_utils.strftime(singer_utils.strptime_with_tz(sync_start_date) - datetime.timedelta(seconds=self.lookback_window)) + sync_start_date = singer_utils.strftime( + singer_utils.strptime_with_tz(sync_start_date) + - datetime.timedelta(seconds=self.lookback_window) + ) return sync_start_date - def _build_query_string(self, catalog_entry, start_date, end_date=None, order_by_clause=True): + def _build_query_string( + self, catalog_entry, start_date, end_date=None, order_by_clause=True + ): selected_properties = self._get_selected_properties(catalog_entry) - query = "SELECT {} FROM {}".format(",".join(selected_properties), catalog_entry['stream']) + query = "SELECT {} FROM {}".format( + ",".join(selected_properties), catalog_entry["stream"] + ) - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_key = catalog_metadata.get((), {}).get('replication-key') + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_key = catalog_metadata.get((), {}).get("replication-key") if replication_key: - where_clause = " WHERE {} >= {} ".format( - replication_key, - start_date) + where_clause = " WHERE {} >= {} ".format(replication_key, start_date) if end_date: end_date_clause = " AND {} < {}".format(replication_key, end_date) else: @@ -444,30 +520,38 @@ def query(self, catalog_entry, state): return rest.query(catalog_entry, state) else: raise TapSalesforceException( - "api_type should be REST or BULK was: {}".format( - self.api_type)) + "api_type should be REST or BULK was: {}".format(self.api_type) + ) def get_blacklisted_objects(self): if self.api_type == BULK_API_TYPE: return UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS.union( - QUERY_RESTRICTED_SALESFORCE_OBJECTS).union(QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS) + QUERY_RESTRICTED_SALESFORCE_OBJECTS + ).union(QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS) elif self.api_type == REST_API_TYPE: - return QUERY_RESTRICTED_SALESFORCE_OBJECTS.union(QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS) + return QUERY_RESTRICTED_SALESFORCE_OBJECTS.union( + QUERY_INCOMPATIBLE_SALESFORCE_OBJECTS + ) else: raise TapSalesforceException( - "api_type should be REST or BULK was: {}".format( - self.api_type)) + "api_type should be REST or BULK was: {}".format(self.api_type) + ) # pylint: disable=line-too-long def get_blacklisted_fields(self): if self.api_type == BULK_API_TYPE: - return {('EntityDefinition', 'RecordTypesSupported'): "this field is unsupported by the Bulk API."} + return { + ( + "EntityDefinition", + "RecordTypesSupported", + ): "this field is unsupported by the Bulk API." + } elif self.api_type == REST_API_TYPE: return {} else: raise TapSalesforceException( - "api_type should be REST or BULK was: {}".format( - self.api_type)) + "api_type should be REST or BULK was: {}".format(self.api_type) + ) def get_window_end_date(self, start_date, end_date): # to update end_date, substract 'half_day_range' (i.e. half of the days between start_date and end_date) @@ -476,6 +560,7 @@ def get_window_end_date(self, start_date, end_date): if half_day_range.days == 0: raise TapSalesforceException( - "Attempting to query by 0 day range, this would cause infinite looping.") + "Attempting to query by 0 day range, this would cause infinite looping." + ) return end_date - half_day_range diff --git a/tap_salesforce/salesforce/bulk.py b/tap_salesforce/salesforce/bulk.py index 157f9950..d756fee5 100644 --- a/tap_salesforce/salesforce/bulk.py +++ b/tap_salesforce/salesforce/bulk.py @@ -2,46 +2,48 @@ import csv import json import sys -import time import tempfile +import time + +import requests import singer import singer.utils as singer_utils -from singer import metrics -import requests -from requests.exceptions import RequestException - import xmltodict +from requests.exceptions import RequestException +from singer import metrics -from tap_salesforce.salesforce.rest import API_VERSION from tap_salesforce.salesforce.exceptions import ( - TapSalesforceException, TapSalesforceQuotaExceededException) + TapSalesforceException, + TapSalesforceQuotaExceededException, +) +from tap_salesforce.salesforce.rest import API_VERSION BATCH_STATUS_POLLING_SLEEP = 20 PK_CHUNKED_BATCH_STATUS_POLLING_SLEEP = 60 ITER_CHUNK_SIZE = 1024 -DEFAULT_CHUNK_SIZE = 100000 # Max is 250000 +DEFAULT_CHUNK_SIZE = 100000 # Max is 250000 MAX_RETRIES = 4 LOGGER = singer.get_logger() + # pylint: disable=inconsistent-return-statements def find_parent(stream): parent_stream = stream if stream.endswith("CleanInfo"): - parent_stream = stream[:stream.find("CleanInfo")] + parent_stream = stream[: stream.find("CleanInfo")] elif stream.endswith("FieldHistory"): - parent_stream = stream[:stream.find("FieldHistory")] + parent_stream = stream[: stream.find("FieldHistory")] elif stream.endswith("History"): - parent_stream = stream[:stream.find("History")] + parent_stream = stream[: stream.find("History")] # If the stripped stream ends with "__" we can assume the parent is a custom table if parent_stream.endswith("__"): - parent_stream += 'c' + parent_stream += "c" return parent_stream -class Bulk(): - +class Bulk: bulk_url = "{}/services/async/{}.0/{}" def __init__(self, sf): @@ -55,7 +57,7 @@ def has_permissions(self): except requests.exceptions.HTTPError as err: if err.response is not None: for error_response_item in err.response.json(): - if error_response_item.get('errorCode') == 'API_DISABLED_FOR_ORG': + if error_response_item.get("errorCode") == "API_DISABLED_FOR_ORG": return False return True @@ -73,39 +75,53 @@ def check_bulk_quota_usage(self): url = self.sf.data_url.format(self.sf.instance_url, API_VERSION, endpoint) with metrics.http_request_timer(endpoint): - resp = self.sf._make_request('GET', url, headers=self.sf._get_standard_headers()).json() + resp = self.sf._make_request( + "GET", url, headers=self.sf._get_standard_headers() + ).json() - quota_max = resp['DailyBulkApiBatches']['Max'] + quota_max = resp["DailyBulkApiBatches"]["Max"] max_requests_for_run = int((self.sf.quota_percent_per_run * quota_max) / 100) - quota_remaining = resp['DailyBulkApiBatches']['Remaining'] + quota_remaining = resp["DailyBulkApiBatches"]["Remaining"] percent_used = (1 - (quota_remaining / quota_max)) * 100 if percent_used > self.sf.quota_percent_total: - total_message = ("Salesforce has reported {}/{} ({:3.2f}%) total Bulk API quota " + - "used across all Salesforce Applications. Terminating " + - "replication to not continue past configured percentage " + - "of {}% total quota.").format(quota_max - quota_remaining, - quota_max, - percent_used, - self.sf.quota_percent_total) + total_message = ( + "Salesforce has reported {}/{} ({:3.2f}%) total Bulk API quota " + + "used across all Salesforce Applications. Terminating " + + "replication to not continue past configured percentage " + + "of {}% total quota." + ).format( + quota_max - quota_remaining, + quota_max, + percent_used, + self.sf.quota_percent_total, + ) raise TapSalesforceQuotaExceededException(total_message) elif self.sf.jobs_completed > max_requests_for_run: - partial_message = ("This replication job has completed {} Bulk API jobs ({:3.2f}% of " + - "total quota). Terminating replication due to allotted " + - "quota of {}% per replication.").format(self.sf.jobs_completed, - (self.sf.jobs_completed / quota_max) * 100, - self.sf.quota_percent_per_run) + partial_message = ( + "This replication job has completed {} Bulk API jobs ({:3.2f}% of " + + "total quota). Terminating replication due to allotted " + + "quota of {}% per replication." + ).format( + self.sf.jobs_completed, + (self.sf.jobs_completed / quota_max) * 100, + self.sf.quota_percent_per_run, + ) raise TapSalesforceQuotaExceededException(partial_message) def _get_bulk_headers(self): - return {"X-SFDC-Session": self.sf.access_token, - "Content-Type": "application/json"} + return { + "X-SFDC-Session": self.sf.access_token, + "Content-Type": "application/json", + } def _can_pk_chunk_job(self, failure_message): - return "QUERY_TIMEOUT" in failure_message or \ - "Retried more" in failure_message or \ - "Failed to write query result" in failure_message + return ( + "QUERY_TIMEOUT" in failure_message + or "Retried more" in failure_message + or "Failed to write query result" in failure_message + ) def _bulk_query(self, catalog_entry, state): job_id = self._create_job(catalog_entry) @@ -117,32 +133,48 @@ def _bulk_query(self, catalog_entry, state): batch_status = self._poll_on_batch_status(job_id, batch_id) - if batch_status['state'] == 'Failed': - if self._can_pk_chunk_job(batch_status['stateMessage']): + if batch_status["state"] == "Failed": + if self._can_pk_chunk_job(batch_status["stateMessage"]): # Get list of batch_status with pk_chunking or date_windowing status_list = self._bulk_with_window([], catalog_entry, start_date) for batch_status in status_list: - job_id = batch_status['job_id'] + job_id = batch_status["job_id"] # Set pk_chunking to True to indicate that we should write a bookmark differently self.sf.pk_chunking = True # Add the bulk Job ID and its batches to the state so it can be resumed if necessary - tap_stream_id = catalog_entry['tap_stream_id'] - state = singer.write_bookmark(state, tap_stream_id, 'JobID', job_id) - state = singer.write_bookmark(state, tap_stream_id, 'BatchIDs', batch_status['completed'][:]) - - for completed_batch_id in batch_status['completed']: - for result in self.get_batch_results(job_id, completed_batch_id, catalog_entry): + tap_stream_id = catalog_entry["tap_stream_id"] + state = singer.write_bookmark(state, tap_stream_id, "JobID", job_id) + state = singer.write_bookmark( + state, tap_stream_id, "BatchIDs", batch_status["completed"][:] + ) + + for completed_batch_id in batch_status["completed"]: + for result in self.get_batch_results( + job_id, completed_batch_id, catalog_entry + ): yield result # Remove the completed batch ID and write state - state['bookmarks'][catalog_entry['tap_stream_id']]["BatchIDs"].remove(completed_batch_id) - LOGGER.info("Finished syncing batch %s. Removed batch from state.", completed_batch_id) - LOGGER.info("Batches to go: %d", len(state['bookmarks'][catalog_entry['tap_stream_id']]["BatchIDs"])) + state["bookmarks"][catalog_entry["tap_stream_id"]][ + "BatchIDs" + ].remove(completed_batch_id) + LOGGER.info( + "Finished syncing batch %s. Removed batch from state.", + completed_batch_id, + ) + LOGGER.info( + "Batches to go: %d", + len( + state["bookmarks"][catalog_entry["tap_stream_id"]][ + "BatchIDs" + ] + ), + ) singer.write_state(state) else: - raise TapSalesforceException(batch_status['stateMessage']) + raise TapSalesforceException(batch_status["stateMessage"]) else: for result in self.get_batch_results(job_id, batch_id, catalog_entry): yield result @@ -156,7 +188,7 @@ def _bulk_query_with_pk_chunking(self, catalog_entry, start_date): self._add_batch(catalog_entry, job_id, start_date, order_by_clause=False) batch_status = self._poll_on_pk_chunked_batch_status(job_id) - batch_status['job_id'] = job_id + batch_status["job_id"] = job_id # Close the job after all the batches are complete self._close_job(job_id) @@ -165,74 +197,93 @@ def _bulk_query_with_pk_chunking(self, catalog_entry, start_date): def _create_job(self, catalog_entry, pk_chunking=False): url = self.bulk_url.format(self.sf.instance_url, API_VERSION, "job") - body = {"operation": "queryAll", "object": catalog_entry['stream'], "contentType": "CSV"} + body = { + "operation": "queryAll", + "object": catalog_entry["stream"], + "contentType": "CSV", + } headers = self._get_bulk_headers() - headers['Sforce-Disable-Batch-Retry'] = "true" + headers["Sforce-Disable-Batch-Retry"] = "true" if pk_chunking: LOGGER.info("ADDING PK CHUNKING HEADER") - headers['Sforce-Enable-PKChunking'] = "true; chunkSize={}".format(DEFAULT_CHUNK_SIZE) + headers["Sforce-Enable-PKChunking"] = "true; chunkSize={}".format( + DEFAULT_CHUNK_SIZE + ) # If the stream ends with 'CleanInfo' or 'History', we can PK Chunk on the object's parent - if any(catalog_entry['stream'].endswith(suffix) for suffix in ["CleanInfo", "History"]): - parent = find_parent(catalog_entry['stream']) - headers['Sforce-Enable-PKChunking'] = headers['Sforce-Enable-PKChunking'] + "; parent={}".format(parent) + if any( + catalog_entry["stream"].endswith(suffix) + for suffix in ["CleanInfo", "History"] + ): + parent = find_parent(catalog_entry["stream"]) + headers["Sforce-Enable-PKChunking"] = headers[ + "Sforce-Enable-PKChunking" + ] + "; parent={}".format(parent) with metrics.http_request_timer("create_job") as timer: - timer.tags['sobject'] = catalog_entry['stream'] + timer.tags["sobject"] = catalog_entry["stream"] resp = self.sf._make_request( - 'POST', - url, - headers=headers, - body=json.dumps(body)) + "POST", url, headers=headers, body=json.dumps(body) + ) job = resp.json() - return job['id'] + return job["id"] - #pylint: disable=too-many-positional-arguments - def _add_batch(self, catalog_entry, job_id, start_date, end_date=None, order_by_clause=True): + # pylint: disable=too-many-positional-arguments + def _add_batch( + self, catalog_entry, job_id, start_date, end_date=None, order_by_clause=True + ): endpoint = "job/{}/batch".format(job_id) url = self.bulk_url.format(self.sf.instance_url, API_VERSION, endpoint) - body = self.sf._build_query_string(catalog_entry, start_date, end_date, order_by_clause=order_by_clause) + body = self.sf._build_query_string( + catalog_entry, start_date, end_date, order_by_clause=order_by_clause + ) headers = self._get_bulk_headers() - headers['Content-Type'] = 'text/csv' + headers["Content-Type"] = "text/csv" with metrics.http_request_timer("add_batch") as timer: - timer.tags['sobject'] = catalog_entry['stream'] - resp = self.sf._make_request('POST', url, headers=headers, body=body) + timer.tags["sobject"] = catalog_entry["stream"] + resp = self.sf._make_request("POST", url, headers=headers, body=body) batch = xmltodict.parse(resp.text) - return batch['batchInfo']['id'] + return batch["batchInfo"]["id"] def _poll_on_pk_chunked_batch_status(self, job_id): batches = self._get_batches(job_id) while True: - queued_batches = [b['id'] for b in batches if b['state'] == "Queued"] - in_progress_batches = [b['id'] for b in batches if b['state'] == "InProgress"] + queued_batches = [b["id"] for b in batches if b["state"] == "Queued"] + in_progress_batches = [ + b["id"] for b in batches if b["state"] == "InProgress" + ] if not queued_batches and not in_progress_batches: - completed_batches = [b['id'] for b in batches if b['state'] == "Completed"] - failed_batches = {b['id']: b.get('stateMessage') for b in batches if b['state'] == "Failed"} - return {'completed': completed_batches, 'failed': failed_batches} + completed_batches = [ + b["id"] for b in batches if b["state"] == "Completed" + ] + failed_batches = { + b["id"]: b.get("stateMessage") + for b in batches + if b["state"] == "Failed" + } + return {"completed": completed_batches, "failed": failed_batches} else: time.sleep(PK_CHUNKED_BATCH_STATUS_POLLING_SLEEP) batches = self._get_batches(job_id) def _poll_on_batch_status(self, job_id, batch_id): - batch_status = self._get_batch(job_id=job_id, - batch_id=batch_id) + batch_status = self._get_batch(job_id=job_id, batch_id=batch_id) - while batch_status['state'] not in ['Completed', 'Failed', 'Not Processed']: + while batch_status["state"] not in ["Completed", "Failed", "Not Processed"]: time.sleep(BATCH_STATUS_POLLING_SLEEP) - batch_status = self._get_batch(job_id=job_id, - batch_id=batch_id) + batch_status = self._get_batch(job_id=job_id, batch_id=batch_id) return batch_status @@ -243,14 +294,14 @@ def job_exists(self, job_id): headers = self._get_bulk_headers() with metrics.http_request_timer("get_job"): - self.sf._make_request('GET', url, headers=headers) + self.sf._make_request("GET", url, headers=headers) - return True # requests will raise for a 400 InvalidJob + return True # requests will raise for a 400 InvalidJob except RequestException as ex: - if ex.response.headers["Content-Type"] == 'application/json': - exception_code = ex.response.json()['exceptionCode'] - if exception_code == 'InvalidJob': + if ex.response.headers["Content-Type"] == "application/json": + exception_code = ex.response.json()["exceptionCode"] + if exception_code == "InvalidJob": return False raise @@ -260,11 +311,11 @@ def _get_batches(self, job_id): headers = self._get_bulk_headers() with metrics.http_request_timer("get_batches"): - resp = self.sf._make_request('GET', url, headers=headers) + resp = self.sf._make_request("GET", url, headers=headers) - batches = xmltodict.parse(resp.text, - xml_attribs=False, - force_list=('batchInfo',))['batchInfoList']['batchInfo'] + batches = xmltodict.parse( + resp.text, xml_attribs=False, force_list=("batchInfo",) + )["batchInfoList"]["batchInfo"] return batches @@ -274,11 +325,11 @@ def _get_batch(self, job_id, batch_id): headers = self._get_bulk_headers() with metrics.http_request_timer("get_batch"): - resp = self.sf._make_request('GET', url, headers=headers) + resp = self.sf._make_request("GET", url, headers=headers) batch = xmltodict.parse(resp.text) - return batch['batchInfo'] + return batch["batchInfo"] def get_batch_results(self, job_id, batch_id, catalog_entry): """Given a job_id and batch_id, queries the batches results and reads @@ -288,32 +339,32 @@ def get_batch_results(self, job_id, batch_id, catalog_entry): url = self.bulk_url.format(self.sf.instance_url, API_VERSION, endpoint) with metrics.http_request_timer("batch_result_list") as timer: - timer.tags['sobject'] = catalog_entry['stream'] - batch_result_resp = self.sf._make_request('GET', url, headers=headers) + timer.tags["sobject"] = catalog_entry["stream"] + batch_result_resp = self.sf._make_request("GET", url, headers=headers) # Returns a Dict where input: # 12 # will return: {'result', ['1', '2']} - batch_result_list = xmltodict.parse(batch_result_resp.text, - xml_attribs=False, - force_list={'result'})['result-list'] + batch_result_list = xmltodict.parse( + batch_result_resp.text, xml_attribs=False, force_list={"result"} + )["result-list"] - for result in batch_result_list['result']: + for result in batch_result_list["result"]: endpoint = "job/{}/batch/{}/result/{}".format(job_id, batch_id, result) url = self.bulk_url.format(self.sf.instance_url, API_VERSION, endpoint) - headers['Content-Type'] = 'text/csv' + headers["Content-Type"] = "text/csv" with tempfile.NamedTemporaryFile(mode="w+", encoding="utf8") as csv_file: - resp = self.sf._make_request('GET', url, headers=headers, stream=True) - for chunk in resp.iter_content(chunk_size=ITER_CHUNK_SIZE, decode_unicode=True): + resp = self.sf._make_request("GET", url, headers=headers, stream=True) + for chunk in resp.iter_content( + chunk_size=ITER_CHUNK_SIZE, decode_unicode=True + ): if chunk: # Replace any NULL bytes in the chunk so it can be safely given to the CSV reader - csv_file.write(chunk.replace('\0', '')) + csv_file.write(chunk.replace("\0", "")) csv_file.seek(0) - csv_reader = csv.reader(csv_file, - delimiter=',', - quotechar='"') + csv_reader = csv.reader(csv_file, delimiter=",", quotechar='"') column_name_list = next(csv_reader) @@ -328,10 +379,8 @@ def _close_job(self, job_id): with metrics.http_request_timer("close_job"): self.sf._make_request( - 'POST', - url, - headers=self._get_bulk_headers(), - body=json.dumps(body)) + "POST", url, headers=self._get_bulk_headers(), body=json.dumps(body) + ) def _iter_lines(self, response): """Clone of the iter_lines function from the requests library with the change @@ -339,7 +388,9 @@ def _iter_lines(self, response): from within a quoted value from the CSV stream.""" pending = None - for chunk in response.iter_content(decode_unicode=True, chunk_size=ITER_CHUNK_SIZE): + for chunk in response.iter_content( + decode_unicode=True, chunk_size=ITER_CHUNK_SIZE + ): if pending is not None: chunk = pending + chunk @@ -356,31 +407,60 @@ def _iter_lines(self, response): if pending is not None: yield pending - def _bulk_with_window(self, status_list, catalog_entry, start_date_str, end_date=None, retries=MAX_RETRIES): + def _bulk_with_window( + self, + status_list, + catalog_entry, + start_date_str, + end_date=None, + retries=MAX_RETRIES, + ): """Bulk api call with date windowing""" sync_start = singer_utils.now() if end_date is None: end_date = sync_start LOGGER.info("Retrying Bulk Query with PK Chunking") else: - LOGGER.info("Retrying Bulk Query with window of date {} to {}".format(start_date_str, end_date.strftime('%Y-%m-%dT%H:%M:%SZ'))) + LOGGER.info( + "Retrying Bulk Query with window of date {} to {}".format( + start_date_str, end_date.strftime("%Y-%m-%dT%H:%M:%SZ") + ) + ) if retries == 0: - raise TapSalesforceException("Ran out of retries attempting to query Salesforce Object {}".format(catalog_entry['stream'])) + raise TapSalesforceException( + "Ran out of retries attempting to query Salesforce Object {}".format( + catalog_entry["stream"] + ) + ) job_id = self._create_job(catalog_entry, True) - self._add_batch(catalog_entry, job_id, start_date_str, end_date.strftime('%Y-%m-%dT%H:%M:%SZ'), False) + self._add_batch( + catalog_entry, + job_id, + start_date_str, + end_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + False, + ) batch_status = self._poll_on_pk_chunked_batch_status(job_id) - batch_status['job_id'] = job_id + batch_status["job_id"] = job_id # Close the job after all the batches are complete self._close_job(job_id) - if batch_status['failed']: - LOGGER.info("Failed Bulk Query with window of date {} to {}".format(start_date_str, end_date.strftime('%Y-%m-%dT%H:%M:%SZ'))) + if batch_status["failed"]: + LOGGER.info( + "Failed Bulk Query with window of date {} to {}".format( + start_date_str, end_date.strftime("%Y-%m-%dT%H:%M:%SZ") + ) + ) # If batch_status is failed then reduce date window by half by updating end_date - end_date = self.sf.get_window_end_date(singer_utils.strptime_with_tz(start_date_str), end_date) + end_date = self.sf.get_window_end_date( + singer_utils.strptime_with_tz(start_date_str), end_date + ) - return self._bulk_with_window(status_list, catalog_entry, start_date_str, end_date, retries - 1) + return self._bulk_with_window( + status_list, catalog_entry, start_date_str, end_date, retries - 1 + ) else: status_list.append(batch_status) @@ -388,7 +468,9 @@ def _bulk_with_window(self, status_list, catalog_entry, start_date_str, end_date # If the date range was chunked (an end_date was passed), sync # from the end_date -> now if end_date < sync_start: - next_start_date_str = end_date.strftime('%Y-%m-%dT%H:%M:%SZ') - return self._bulk_with_window(status_list, catalog_entry, next_start_date_str, retries=retries) + next_start_date_str = end_date.strftime("%Y-%m-%dT%H:%M:%SZ") + return self._bulk_with_window( + status_list, catalog_entry, next_start_date_str, retries=retries + ) return status_list diff --git a/tap_salesforce/salesforce/rest.py b/tap_salesforce/salesforce/rest.py index a8807fef..08f3c2aa 100644 --- a/tap_salesforce/salesforce/rest.py +++ b/tap_salesforce/salesforce/rest.py @@ -2,14 +2,15 @@ import singer import singer.utils as singer_utils from requests.exceptions import HTTPError + from tap_salesforce.salesforce.exceptions import TapSalesforceException LOGGER = singer.get_logger() -API_VERSION = '61' +API_VERSION = "61" MAX_RETRIES = 4 -class Rest(): +class Rest: def __init__(self, sf): self.sf = sf @@ -21,14 +22,12 @@ def query(self, catalog_entry, state): # pylint: disable=too-many-positional-arguments def _query_recur( - self, - query, - catalog_entry, - start_date_str, - end_date=None, - retries=MAX_RETRIES): + self, query, catalog_entry, start_date_str, end_date=None, retries=MAX_RETRIES + ): params = {"q": query} - url = "{}/services/data/v{}.0/queryAll".format(self.sf.instance_url, API_VERSION) + url = "{}/services/data/v{}.0/queryAll".format( + self.sf.instance_url, API_VERSION + ) headers = self.sf._get_standard_headers() sync_start = singer_utils.now() @@ -38,7 +37,9 @@ def _query_recur( if retries == 0: raise TapSalesforceException( "Ran out of retries attempting to query Salesforce Object {}".format( - catalog_entry['stream'])) + catalog_entry["stream"] + ) + ) retryable = False try: @@ -51,47 +52,51 @@ def _query_recur( next_start_date_str = singer_utils.strftime(end_date) query = self.sf._build_query_string(catalog_entry, next_start_date_str) for record in self._query_recur( - query, - catalog_entry, - next_start_date_str, - retries=retries): + query, catalog_entry, next_start_date_str, retries=retries + ): yield record except HTTPError as ex: response = ex.response.json() - if isinstance(response, list) and response[0].get("errorCode") == "QUERY_TIMEOUT": + if ( + isinstance(response, list) + and response[0].get("errorCode") == "QUERY_TIMEOUT" + ): start_date = singer_utils.strptime_with_tz(start_date_str) day_range = (end_date - start_date).days LOGGER.info( "Salesforce returned QUERY_TIMEOUT querying %d days of %s", day_range, - catalog_entry['stream']) + catalog_entry["stream"], + ) retryable = True else: raise ex if retryable: - end_date = self.sf.get_window_end_date(singer_utils.strptime_with_tz(start_date_str), end_date) - - query = self.sf._build_query_string(catalog_entry, singer_utils.strftime(start_date), - singer_utils.strftime(end_date)) + end_date = self.sf.get_window_end_date( + singer_utils.strptime_with_tz(start_date_str), end_date + ) + + query = self.sf._build_query_string( + catalog_entry, + singer_utils.strftime(start_date), + singer_utils.strftime(end_date), + ) for record in self._query_recur( - query, - catalog_entry, - start_date_str, - end_date, - retries - 1): + query, catalog_entry, start_date_str, end_date, retries - 1 + ): yield record def _sync_records(self, url, headers, params): while True: - resp = self.sf._make_request('GET', url, headers=headers, params=params) + resp = self.sf._make_request("GET", url, headers=headers, params=params) resp_json = resp.json() - for rec in resp_json.get('records'): + for rec in resp_json.get("records"): yield rec - next_records_url = resp_json.get('nextRecordsUrl') + next_records_url = resp_json.get("nextRecordsUrl") if next_records_url is None: break diff --git a/tap_salesforce/sync.py b/tap_salesforce/sync.py index 5cea0263..3f5d9db9 100644 --- a/tap_salesforce/sync.py +++ b/tap_salesforce/sync.py @@ -1,18 +1,21 @@ import time + import singer import singer.utils as singer_utils -from singer import Transformer, metadata, metrics -from singer import SingerSyncError from requests.exceptions import RequestException +from singer import SingerSyncError, Transformer, metadata, metrics + from tap_salesforce.salesforce.bulk import Bulk LOGGER = singer.get_logger() -BLACKLISTED_FIELDS = set(['attributes']) +BLACKLISTED_FIELDS = set(["attributes"]) + def remove_blacklisted_fields(data): return {k: v for k, v in data.items() if k not in BLACKLISTED_FIELDS} + # pylint: disable=unused-argument def transform_bulk_data_hook(data, typ, schema): result = data @@ -22,47 +25,53 @@ def transform_bulk_data_hook(data, typ, schema): # Salesforce can return the value '0.0' for integer typed fields. This # causes a schema violation. Convert it to '0' if schema['type'] has # integer. - if data == '0.0' and 'integer' in schema.get('type', []): - result = '0' + if data == "0.0" and "integer" in schema.get("type", []): + result = "0" # Salesforce Bulk API returns CSV's with empty strings for text fields. # When the text field is nillable and the data value is an empty string, # change the data so that it is None. - if data == "" and "null" in schema['type']: + if data == "" and "null" in schema["type"]: result = None return result + def get_stream_version(catalog_entry, state): - tap_stream_id = catalog_entry['tap_stream_id'] - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_key = catalog_metadata.get((), {}).get('replication-key') + tap_stream_id = catalog_entry["tap_stream_id"] + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_key = catalog_metadata.get((), {}).get("replication-key") - if singer.get_bookmark(state, tap_stream_id, 'version') is None: + if singer.get_bookmark(state, tap_stream_id, "version") is None: stream_version = int(time.time() * 1000) else: - stream_version = singer.get_bookmark(state, tap_stream_id, 'version') + stream_version = singer.get_bookmark(state, tap_stream_id, "version") if replication_key: return stream_version return int(time.time() * 1000) + def resume_syncing_bulk_query(sf, catalog_entry, job_id, state, counter): bulk = Bulk(sf) - current_bookmark = singer.get_bookmark(state, catalog_entry['tap_stream_id'], 'JobHighestBookmarkSeen') or sf.get_start_date(state, catalog_entry) + current_bookmark = singer.get_bookmark( + state, catalog_entry["tap_stream_id"], "JobHighestBookmarkSeen" + ) or sf.get_start_date(state, catalog_entry) current_bookmark = singer_utils.strptime_with_tz(current_bookmark) - batch_ids = singer.get_bookmark(state, catalog_entry['tap_stream_id'], 'BatchIDs') + batch_ids = singer.get_bookmark(state, catalog_entry["tap_stream_id"], "BatchIDs") start_time = singer_utils.now() - stream = catalog_entry['stream'] - stream_alias = catalog_entry.get('stream_alias') - catalog_metadata = metadata.to_map(catalog_entry.get('metadata')) - replication_key = catalog_metadata.get((), {}).get('replication-key') + stream = catalog_entry["stream"] + stream_alias = catalog_entry.get("stream_alias") + catalog_metadata = metadata.to_map(catalog_entry.get("metadata")) + replication_key = catalog_metadata.get((), {}).get("replication-key") stream_version = get_stream_version(catalog_entry, state) - schema = catalog_entry['schema'] + schema = catalog_entry["schema"] if not bulk.job_exists(job_id): - LOGGER.info("Found stored Job ID that no longer exists, resetting bookmark and removing JobID from state.") + LOGGER.info( + "Found stored Job ID that no longer exists, resetting bookmark and removing JobID from state." + ) return counter # Iterate over the remaining batches, removing them once they are synced @@ -74,21 +83,33 @@ def resume_syncing_bulk_query(sf, catalog_entry, job_id, state, counter): rec = fix_record_anytype(rec, schema) singer.write_message( singer.RecordMessage( - stream=( - stream_alias or stream), + stream=(stream_alias or stream), record=rec, version=stream_version, - time_extracted=start_time)) + time_extracted=start_time, + ) + ) # Update bookmark if necessary - replication_key_value = replication_key and singer_utils.strptime_with_tz(rec[replication_key]) - if replication_key_value and replication_key_value <= start_time and replication_key_value > current_bookmark: - current_bookmark = singer_utils.strptime_with_tz(rec[replication_key]) - - state = singer.write_bookmark(state, - catalog_entry['tap_stream_id'], - 'JobHighestBookmarkSeen', - singer_utils.strftime(current_bookmark)) + replication_key_value = ( + replication_key + and singer_utils.strptime_with_tz(rec[replication_key]) + ) + if ( + replication_key_value + and replication_key_value <= start_time + and replication_key_value > current_bookmark + ): + current_bookmark = singer_utils.strptime_with_tz( + rec[replication_key] + ) + + state = singer.write_bookmark( + state, + catalog_entry["tap_stream_id"], + "JobHighestBookmarkSeen", + singer_utils.strftime(current_bookmark), + ) batch_ids.remove(batch_id) LOGGER.info("Finished syncing batch %s. Removing batch from state.", batch_id) LOGGER.info("Batches to go: %d", len(batch_ids)) @@ -96,40 +117,47 @@ def resume_syncing_bulk_query(sf, catalog_entry, job_id, state, counter): return counter + def sync_stream(sf, catalog_entry, state): - stream = catalog_entry['stream'] + stream = catalog_entry["stream"] with metrics.record_counter(stream) as counter: try: sync_records(sf, catalog_entry, state, counter) singer.write_state(state) except RequestException as ex: - raise Exception("{} Response: {}, (Stream: {})".format( - ex, ex.response.text, stream)) from ex + raise Exception( + "{} Response: {}, (Stream: {})".format(ex, ex.response.text, stream) + ) from ex except Exception as ex: if "OPERATION_TOO_LARGE: exceeded 100000 distinct who/what's" in str(ex): - raise SingerSyncError("OPERATION_TOO_LARGE: exceeded 100000 distinct who/what's. " + - "Consider asking your Salesforce System Administrator to provide you with the " + - "`View All Data` profile permission. (Stream: {})".format(stream)) from ex - raise Exception("{}, (Stream: {})".format( - ex, stream)) from ex + raise SingerSyncError( + "OPERATION_TOO_LARGE: exceeded 100000 distinct who/what's. " + + "Consider asking your Salesforce System Administrator to provide you with the " + + "`View All Data` profile permission. (Stream: {})".format(stream) + ) from ex + raise Exception("{}, (Stream: {})".format(ex, stream)) from ex return counter + def sync_records(sf, catalog_entry, state, counter): - chunked_bookmark = singer_utils.strptime_with_tz(sf.get_start_date(state, catalog_entry)) - stream = catalog_entry['stream'] - schema = catalog_entry['schema'] - stream_alias = catalog_entry.get('stream_alias') - catalog_metadata = metadata.to_map(catalog_entry['metadata']) - replication_key = catalog_metadata.get((), {}).get('replication-key') + chunked_bookmark = singer_utils.strptime_with_tz( + sf.get_start_date(state, catalog_entry) + ) + stream = catalog_entry["stream"] + schema = catalog_entry["schema"] + stream_alias = catalog_entry.get("stream_alias") + catalog_metadata = metadata.to_map(catalog_entry["metadata"]) + replication_key = catalog_metadata.get((), {}).get("replication-key") stream_version = get_stream_version(catalog_entry, state) - activate_version_message = singer.ActivateVersionMessage(stream=(stream_alias or stream), - version=stream_version) + activate_version_message = singer.ActivateVersionMessage( + stream=(stream_alias or stream), version=stream_version + ) start_time = singer_utils.now() - LOGGER.info('Syncing Salesforce data for stream %s', stream) + LOGGER.info("Syncing Salesforce data for stream %s", stream) for rec in sf.query(catalog_entry, state): counter.increment() @@ -138,32 +166,41 @@ def sync_records(sf, catalog_entry, state, counter): rec = fix_record_anytype(rec, schema) singer.write_message( singer.RecordMessage( - stream=( - stream_alias or stream), + stream=(stream_alias or stream), record=rec, version=stream_version, - time_extracted=start_time)) + time_extracted=start_time, + ) + ) - replication_key_value = replication_key and singer_utils.strptime_with_tz(rec[replication_key]) + replication_key_value = replication_key and singer_utils.strptime_with_tz( + rec[replication_key] + ) if sf.pk_chunking: - if replication_key_value and replication_key_value <= start_time and replication_key_value > chunked_bookmark: + if ( + replication_key_value + and replication_key_value <= start_time + and replication_key_value > chunked_bookmark + ): # Replace the highest seen bookmark and save the state in case we need to resume later chunked_bookmark = singer_utils.strptime_with_tz(rec[replication_key]) state = singer.write_bookmark( state, - catalog_entry['tap_stream_id'], - 'JobHighestBookmarkSeen', - singer_utils.strftime(chunked_bookmark)) + catalog_entry["tap_stream_id"], + "JobHighestBookmarkSeen", + singer_utils.strftime(chunked_bookmark), + ) singer.write_state(state) # Before writing a bookmark, make sure Salesforce has not given us a # record with one outside our range elif replication_key_value and replication_key_value <= start_time: state = singer.write_bookmark( state, - catalog_entry['tap_stream_id'], + catalog_entry["tap_stream_id"], replication_key, - rec[replication_key]) + rec[replication_key], + ) singer.write_state(state) # Tables with no replication_key will send an @@ -171,20 +208,24 @@ def sync_records(sf, catalog_entry, state, counter): if not replication_key: singer.write_message(activate_version_message) state = singer.write_bookmark( - state, catalog_entry['tap_stream_id'], 'version', None) + state, catalog_entry["tap_stream_id"], "version", None + ) # If pk_chunking is set, and selected streams has replication key then only write a bookmark at the end if sf.pk_chunking and replication_key: # Write a bookmark with the highest value we've seen state = singer.write_bookmark( state, - catalog_entry['tap_stream_id'], + catalog_entry["tap_stream_id"], replication_key, - singer_utils.strftime(chunked_bookmark)) + singer_utils.strftime(chunked_bookmark), + ) + def fix_record_anytype(rec, schema): """Modifies a record when the schema has no 'type' element due to a SF type of 'anyType.' Attempts to set the record's value for that element to an int, float, or string.""" + def try_cast(val, coercion): try: return coercion(val) @@ -192,12 +233,12 @@ def try_cast(val, coercion): return val for k, v in rec.items(): - if schema['properties'][k].get("type") is None: + if schema["properties"][k].get("type") is None: val = v val = try_cast(v, int) val = try_cast(v, float) if v in ["true", "false"]: - val = (v == "true") # pylint: disable=superfluous-parens + val = v == "true" # pylint: disable=superfluous-parens if v == "": val = None diff --git a/tests/base.py b/tests/base.py index fd2e8610..587d09df 100644 --- a/tests/base.py +++ b/tests/base.py @@ -2,14 +2,15 @@ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests """ -import unittest + import os -from datetime import timedelta from datetime import datetime as dt +from datetime import timedelta -from tap_tester import connections, menagerie, runner, LOGGER +from tap_tester import LOGGER, connections, menagerie, runner from tap_tester.base_case import BaseCase + class SalesforceBaseTest(BaseCase): """ Setup expectations for test sub classes. @@ -25,6 +26,7 @@ class SalesforceBaseTest(BaseCase): timing | https://stitchdata.atlassian.net/browse/SRCE-4816 more data | https://stitchdata.atlassian.net/browse/SRCE-4824 """ + AUTOMATIC_FIELDS = "automatic" REPLICATION_KEYS = "valid-replication-keys" PRIMARY_KEYS = "table-key-properties" @@ -51,11 +53,11 @@ def get_type(): def get_properties(self, original: bool = True): """Configuration properties required for the tap.""" return_value = { - 'start_date': '2020-11-23T00:00:00Z', - 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', - 'quota_percent_total': '95', - 'api_type': self.salesforce_api, - 'is_sandbox': 'false' + "start_date": "2020-11-23T00:00:00Z", + "instance_url": "https://singer2-dev-ed.my.salesforce.com", + "quota_percent_total": "95", + "api_type": self.salesforce_api, + "is_sandbox": "false", } if original: @@ -70,16 +72,18 @@ def get_properties(self, original: bool = True): @staticmethod def get_credentials(): """Authentication information for the test account""" - return {'refresh_token': os.getenv('TAP_SALESFORCE_REFRESH_TOKEN'), - 'client_id': os.getenv('TAP_SALESFORCE_CLIENT_ID'), - 'client_secret': os.getenv('TAP_SALESFORCE_CLIENT_SECRET')} + return { + "refresh_token": os.getenv("TAP_SALESFORCE_REFRESH_TOKEN"), + "client_id": os.getenv("TAP_SALESFORCE_CLIENT_ID"), + "client_secret": os.getenv("TAP_SALESFORCE_CLIENT_SECRET"), + } def expected_metadata(self): """The expected streams and metadata about the streams""" default = { self.PRIMARY_KEYS: {"Id"}, self.REPLICATION_METHOD: self.INCREMENTAL, - self.REPLICATION_KEYS: {"SystemModstamp"} + self.REPLICATION_KEYS: {"SystemModstamp"}, } default_full = { self.PRIMARY_KEYS: {"Id"}, @@ -87,14 +91,14 @@ def expected_metadata(self): } incremental_created_date = { - self.REPLICATION_KEYS: {'CreatedDate'}, - self.PRIMARY_KEYS: {'Id'}, + self.REPLICATION_KEYS: {"CreatedDate"}, + self.PRIMARY_KEYS: {"Id"}, self.REPLICATION_METHOD: self.INCREMENTAL, } incremental_last_modified = { - self.PRIMARY_KEYS: {'Id'}, - self.REPLICATION_KEYS: {'LastModifiedDate'}, + self.PRIMARY_KEYS: {"Id"}, + self.REPLICATION_KEYS: {"LastModifiedDate"}, self.REPLICATION_METHOD: self.INCREMENTAL, } @@ -104,1030 +108,1034 @@ def expected_metadata(self): } return { - 'AIApplication': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIApplicationConfig': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIInsightAction': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIInsightFeedback': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIInsightReason': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIInsightValue': default, # removed # 6/13/2022 added back 7/10/2022 - 'AIRecordInsight': default, # removed # 6/13/2022 added back 7/10/2022 - 'Account': default, - 'AccountCleanInfo': default, # new - 'AccountContactRole': default, - 'AccountFeed': default, - 'AccountHistory': incremental_created_date, - 'AccountPartner': default, - 'AccountShare': incremental_last_modified, - 'ActionLinkGroupTemplate': default, - 'ActionLinkTemplate': default, - 'ActiveFeatureLicenseMetric': default, - 'ActivePermSetLicenseMetric': default, - 'ActiveProfileMetric': default, - 'AdditionalNumber': default, - 'AlternativePaymentMethod': default, # new - 'AlternativePaymentMethodShare': incremental_last_modified, # new - 'ApexClass': default, - 'ApexComponent': default, - 'ApexEmailNotification': default, # new - 'ApexLog': default, # new - 'ApexPage': default, - 'ApexPageInfo': default_full, - 'ApexTestQueueItem': default, - 'ApexTestResult': default, - 'ApexTestResultLimits': default, - 'ApexTestRunResult': default, - 'ApexTestSuite': default, - 'ApexTrigger': default, - 'ApiAnomalyEventStore': default, # new - 'ApiAnomalyEventStoreFeed': default, # new - 'ApiEvent': default_full, # new - 'AppAnalyticsQueryRequest': default, # new - 'AppDefinition': default_full, - 'AppMenuItem': default, - 'AppUsageAssignment': default, # removed # 6/13/2022 added back 7/10/2022 - 'AppointmentAssignmentPolicy': default, # new - 'AppointmentScheduleAggr': default, # new - 'AppointmentScheduleLog': default, # new - 'AppointmentSchedulingPolicy': default, # new - 'AppointmentTopicTimeSlot': default, # new - 'AppointmentTopicTimeSlotFeed': default, # new - 'AppointmentTopicTimeSlotHistory': incremental_created_date, # new - 'Asset': default, - 'AssetAction': default, # new - 'AssetActionSource': default, # new - 'AssetFeed': default, - 'AssetHistory': incremental_created_date, - 'AssetRelationship': default, - 'AssetRelationshipFeed': default, - 'AssetRelationshipHistory': incremental_created_date, - 'AssetShare': incremental_last_modified, - 'AssetStatePeriod': default, # new - 'AssignedResource': default, # new - 'AssignedResourceFeed': default, # new - 'AssignmentRule': default, - 'AssociatedLocation': default, - 'AssociatedLocationHistory': incremental_created_date, - 'AsyncApexJob': incremental_created_date, - 'AsyncOperationLog': default, # new - 'Attachment': default, - 'AuraDefinition': default, - 'AuraDefinitionBundle': default, - 'AuraDefinitionBundleInfo': default_full, - 'AuraDefinitionInfo': incremental_last_modified, - 'AuthConfig': default, - 'AuthConfigProviders': default, - 'AuthProvider': incremental_created_date, # new - 'AuthSession': incremental_last_modified, - 'AuthorizationForm': default, - 'AuthorizationFormConsent': default, - 'AuthorizationFormConsentHistory': incremental_created_date, - 'AuthorizationFormConsentShare': incremental_last_modified, - 'AuthorizationFormDataUse': default, - 'AuthorizationFormDataUseHistory': incremental_created_date, - 'AuthorizationFormDataUseShare': incremental_last_modified, - 'AuthorizationFormHistory': incremental_created_date, - 'AuthorizationFormShare': incremental_last_modified, - 'AuthorizationFormText': default, - 'AuthorizationFormTextFeed': default, - 'AuthorizationFormTextHistory': incremental_created_date, - 'BackgroundOperation': default, # new - 'BrandTemplate': default, - 'BrandingSet': default, - 'BrandingSetProperty': default, - 'BulkApiResultEventStore': default_full, # new - 'BusinessHours': default, - 'BusinessProcess': default, - 'Calendar': default, - 'CalendarView': default, - 'CalendarViewShare': incremental_last_modified, - 'CallCenter': default, - 'CallCoachingMediaProvider': default, - 'Campaign': default, - 'CampaignFeed': default, - 'CampaignHistory': incremental_created_date, - 'CampaignMember': default, - 'CampaignMemberStatus': default, - 'CampaignShare': incremental_last_modified, - 'CardPaymentMethod': default, # new - 'Case': default, - 'CaseComment': default, - 'CaseContactRole': default, - 'CaseFeed': default, - 'CaseHistory': incremental_created_date, - 'CaseMilestone': default, # new - 'CaseShare': incremental_last_modified, - 'CaseSolution': default, - 'CaseTeamMember': default, - 'CaseTeamRole': default, - 'CaseTeamTemplate': default, - 'CaseTeamTemplateMember': default, - 'CaseTeamTemplateRecord': default, - 'CategoryData': default, # new - 'CategoryNode': default, - 'ChatterActivity': default, - 'ChatterExtension': default, - 'ChatterExtensionConfig': default, - 'ClientBrowser': incremental_created_date, # Removed 09/03/25 # Re-added 10/24/25 - 'CollaborationGroup': default, - 'CollaborationGroupFeed': default, - 'CollaborationGroupMember': default, - 'CollaborationGroupMemberRequest': default, - 'CollaborationInvitation': default, - 'CommSubscription': default, - 'CommSubscriptionChannelType': default, - 'CommSubscriptionChannelTypeFeed': default, - 'CommSubscriptionChannelTypeHistory': incremental_created_date, - 'CommSubscriptionChannelTypeShare': incremental_last_modified, - 'CommSubscriptionConsent': default, - 'CommSubscriptionConsentFeed': default, - 'CommSubscriptionConsentHistory': incremental_created_date, - 'CommSubscriptionConsentShare': incremental_last_modified, - 'CommSubscriptionFeed': default, - 'CommSubscriptionHistory': incremental_created_date, - 'CommSubscriptionShare': incremental_last_modified, - 'CommSubscriptionTiming': default, - 'CommSubscriptionTimingFeed': default, - 'CommSubscriptionTimingHistory': incremental_created_date, - 'Community': default, - 'ConferenceNumber': default, - 'ConnectedApplication': default, - 'ConsumptionRate': default, # new - 'ConsumptionRateHistory': incremental_created_date, # new - 'ConsumptionSchedule': default, # new - 'ConsumptionScheduleFeed': default, # new - 'ConsumptionScheduleHistory': incremental_created_date, # new - 'ConsumptionScheduleShare': incremental_last_modified, # new - 'Contact': default, - 'ContactCleanInfo': default, # new - 'ContactFeed': default, - 'ContactHistory': incremental_created_date, - 'ContactPointAddress': default, - 'ContactPointAddressHistory': incremental_created_date, - 'ContactPointAddressShare': incremental_last_modified, - 'ContactPointConsent': default, - 'ContactPointConsentHistory': incremental_created_date, - 'ContactPointConsentShare': incremental_last_modified, - 'ContactPointEmail': default, - 'ContactPointEmailHistory': incremental_created_date, - 'ContactPointEmailShare': incremental_last_modified, - 'ContactPointPhone': default, - 'ContactPointPhoneHistory': incremental_created_date, - 'ContactPointPhoneShare': incremental_last_modified, - 'ContactPointTypeConsent': default, - 'ContactPointTypeConsentHistory': incremental_created_date, - 'ContactPointTypeConsentShare': incremental_last_modified, - 'ContactRequest': default, # new - 'ContactRequestShare': incremental_last_modified, # new - 'ContactShare': incremental_last_modified, - 'ContentAsset': default, - 'ContentDistribution': default, - 'ContentDistributionView': default, - 'ContentDocument': default, - 'ContentDocumentFeed': default, - 'ContentDocumentHistory': incremental_created_date, - 'ContentDocumentSubscription': default_full, # new - 'ContentFolder': default, - 'ContentFolderLink': default_full, - 'ContentNotification': incremental_created_date, # new - 'ContentTagSubscription': default_full, # new - 'ContentUserSubscription': default_full, # new - 'ContentVersion': default, - 'ContentVersionComment': incremental_created_date, # new - 'ContentVersionHistory': incremental_created_date, - 'ContentVersionRating': incremental_last_modified, # new - 'ContentWorkspace': default, - 'ContentWorkspaceDoc': default, - 'ContentWorkspaceMember': incremental_created_date, - 'ContentWorkspacePermission': default, - 'ContentWorkspaceSubscription': default_full, # new - 'Contract': default, - 'ContractContactRole': default, - 'ContractFeed': default, - 'ContractHistory': incremental_created_date, - 'ContractLineItem': default, # new - 'ContractLineItemHistory': incremental_created_date, # new - 'ConversationEntry': default, # new - 'CorsWhitelistEntry': default, # new - 'CredentialStuffingEventStore': default, # new - 'CredentialStuffingEventStoreFeed': default, # new - 'CreditMemo': default, # new - 'CreditMemoFeed': default, # new - 'CreditMemoHistory': incremental_created_date, # new - 'CreditMemoInvApplication': default, # new - 'CreditMemoInvApplicationFeed': default, # new 6/13/2022 + "AIApplication": default, # removed # 6/13/2022 added back 7/10/2022 + "AIApplicationConfig": default, # removed # 6/13/2022 added back 7/10/2022 + "AIInsightAction": default, # removed # 6/13/2022 added back 7/10/2022 + "AIInsightFeedback": default, # removed # 6/13/2022 added back 7/10/2022 + "AIInsightReason": default, # removed # 6/13/2022 added back 7/10/2022 + "AIInsightValue": default, # removed # 6/13/2022 added back 7/10/2022 + "AIRecordInsight": default, # removed # 6/13/2022 added back 7/10/2022 + "Account": default, + "AccountCleanInfo": default, # new + "AccountContactRole": default, + "AccountFeed": default, + "AccountHistory": incremental_created_date, + "AccountPartner": default, + "AccountShare": incremental_last_modified, + "ActionLinkGroupTemplate": default, + "ActionLinkTemplate": default, + "ActiveFeatureLicenseMetric": default, + "ActivePermSetLicenseMetric": default, + "ActiveProfileMetric": default, + "AdditionalNumber": default, + "AlternativePaymentMethod": default, # new + "AlternativePaymentMethodShare": incremental_last_modified, # new + "ApexClass": default, + "ApexComponent": default, + "ApexEmailNotification": default, # new + "ApexLog": default, # new + "ApexPage": default, + "ApexPageInfo": default_full, + "ApexTestQueueItem": default, + "ApexTestResult": default, + "ApexTestResultLimits": default, + "ApexTestRunResult": default, + "ApexTestSuite": default, + "ApexTrigger": default, + "ApiAnomalyEventStore": default, # new + "ApiAnomalyEventStoreFeed": default, # new + "ApiEvent": default_full, # new + "AppAnalyticsQueryRequest": default, # new + "AppDefinition": default_full, + "AppMenuItem": default, + "AppUsageAssignment": default, # removed # 6/13/2022 added back 7/10/2022 + "AppointmentAssignmentPolicy": default, # new + "AppointmentScheduleAggr": default, # new + "AppointmentScheduleLog": default, # new + "AppointmentSchedulingPolicy": default, # new + "AppointmentTopicTimeSlot": default, # new + "AppointmentTopicTimeSlotFeed": default, # new + "AppointmentTopicTimeSlotHistory": incremental_created_date, # new + "Asset": default, + "AssetAction": default, # new + "AssetActionSource": default, # new + "AssetFeed": default, + "AssetHistory": incremental_created_date, + "AssetRelationship": default, + "AssetRelationshipFeed": default, + "AssetRelationshipHistory": incremental_created_date, + "AssetShare": incremental_last_modified, + "AssetStatePeriod": default, # new + "AssignedResource": default, # new + "AssignedResourceFeed": default, # new + "AssignmentRule": default, + "AssociatedLocation": default, + "AssociatedLocationHistory": incremental_created_date, + "AsyncApexJob": incremental_created_date, + "AsyncOperationLog": default, # new + "Attachment": default, + "AuraDefinition": default, + "AuraDefinitionBundle": default, + "AuraDefinitionBundleInfo": default_full, + "AuraDefinitionInfo": incremental_last_modified, + "AuthConfig": default, + "AuthConfigProviders": default, + "AuthProvider": incremental_created_date, # new + "AuthSession": incremental_last_modified, + "AuthorizationForm": default, + "AuthorizationFormConsent": default, + "AuthorizationFormConsentHistory": incremental_created_date, + "AuthorizationFormConsentShare": incremental_last_modified, + "AuthorizationFormDataUse": default, + "AuthorizationFormDataUseHistory": incremental_created_date, + "AuthorizationFormDataUseShare": incremental_last_modified, + "AuthorizationFormHistory": incremental_created_date, + "AuthorizationFormShare": incremental_last_modified, + "AuthorizationFormText": default, + "AuthorizationFormTextFeed": default, + "AuthorizationFormTextHistory": incremental_created_date, + "BackgroundOperation": default, # new + "BrandTemplate": default, + "BrandingSet": default, + "BrandingSetProperty": default, + "BulkApiResultEventStore": default_full, # new + "BusinessHours": default, + "BusinessProcess": default, + "Calendar": default, + "CalendarView": default, + "CalendarViewShare": incremental_last_modified, + "CallCenter": default, + "CallCoachingMediaProvider": default, + "Campaign": default, + "CampaignFeed": default, + "CampaignHistory": incremental_created_date, + "CampaignMember": default, + "CampaignMemberStatus": default, + "CampaignShare": incremental_last_modified, + "CardPaymentMethod": default, # new + "Case": default, + "CaseComment": default, + "CaseContactRole": default, + "CaseFeed": default, + "CaseHistory": incremental_created_date, + "CaseMilestone": default, # new + "CaseShare": incremental_last_modified, + "CaseSolution": default, + "CaseTeamMember": default, + "CaseTeamRole": default, + "CaseTeamTemplate": default, + "CaseTeamTemplateMember": default, + "CaseTeamTemplateRecord": default, + "CategoryData": default, # new + "CategoryNode": default, + "ChatterActivity": default, + "ChatterExtension": default, + "ChatterExtensionConfig": default, + "ClientBrowser": incremental_created_date, # Removed 09/03/25 # Re-added 10/24/25 + "CollaborationGroup": default, + "CollaborationGroupFeed": default, + "CollaborationGroupMember": default, + "CollaborationGroupMemberRequest": default, + "CollaborationInvitation": default, + "CommSubscription": default, + "CommSubscriptionChannelType": default, + "CommSubscriptionChannelTypeFeed": default, + "CommSubscriptionChannelTypeHistory": incremental_created_date, + "CommSubscriptionChannelTypeShare": incremental_last_modified, + "CommSubscriptionConsent": default, + "CommSubscriptionConsentFeed": default, + "CommSubscriptionConsentHistory": incremental_created_date, + "CommSubscriptionConsentShare": incremental_last_modified, + "CommSubscriptionFeed": default, + "CommSubscriptionHistory": incremental_created_date, + "CommSubscriptionShare": incremental_last_modified, + "CommSubscriptionTiming": default, + "CommSubscriptionTimingFeed": default, + "CommSubscriptionTimingHistory": incremental_created_date, + "Community": default, + "ConferenceNumber": default, + "ConnectedApplication": default, + "ConsumptionRate": default, # new + "ConsumptionRateHistory": incremental_created_date, # new + "ConsumptionSchedule": default, # new + "ConsumptionScheduleFeed": default, # new + "ConsumptionScheduleHistory": incremental_created_date, # new + "ConsumptionScheduleShare": incremental_last_modified, # new + "Contact": default, + "ContactCleanInfo": default, # new + "ContactFeed": default, + "ContactHistory": incremental_created_date, + "ContactPointAddress": default, + "ContactPointAddressHistory": incremental_created_date, + "ContactPointAddressShare": incremental_last_modified, + "ContactPointConsent": default, + "ContactPointConsentHistory": incremental_created_date, + "ContactPointConsentShare": incremental_last_modified, + "ContactPointEmail": default, + "ContactPointEmailHistory": incremental_created_date, + "ContactPointEmailShare": incremental_last_modified, + "ContactPointPhone": default, + "ContactPointPhoneHistory": incremental_created_date, + "ContactPointPhoneShare": incremental_last_modified, + "ContactPointTypeConsent": default, + "ContactPointTypeConsentHistory": incremental_created_date, + "ContactPointTypeConsentShare": incremental_last_modified, + "ContactRequest": default, # new + "ContactRequestShare": incremental_last_modified, # new + "ContactShare": incremental_last_modified, + "ContentAsset": default, + "ContentDistribution": default, + "ContentDistributionView": default, + "ContentDocument": default, + "ContentDocumentFeed": default, + "ContentDocumentHistory": incremental_created_date, + "ContentDocumentSubscription": default_full, # new + "ContentFolder": default, + "ContentFolderLink": default_full, + "ContentNotification": incremental_created_date, # new + "ContentTagSubscription": default_full, # new + "ContentUserSubscription": default_full, # new + "ContentVersion": default, + "ContentVersionComment": incremental_created_date, # new + "ContentVersionHistory": incremental_created_date, + "ContentVersionRating": incremental_last_modified, # new + "ContentWorkspace": default, + "ContentWorkspaceDoc": default, + "ContentWorkspaceMember": incremental_created_date, + "ContentWorkspacePermission": default, + "ContentWorkspaceSubscription": default_full, # new + "Contract": default, + "ContractContactRole": default, + "ContractFeed": default, + "ContractHistory": incremental_created_date, + "ContractLineItem": default, # new + "ContractLineItemHistory": incremental_created_date, # new + "ConversationEntry": default, # new + "CorsWhitelistEntry": default, # new + "CredentialStuffingEventStore": default, # new + "CredentialStuffingEventStoreFeed": default, # new + "CreditMemo": default, # new + "CreditMemoFeed": default, # new + "CreditMemoHistory": incremental_created_date, # new + "CreditMemoInvApplication": default, # new + "CreditMemoInvApplicationFeed": default, # new 6/13/2022 # 'CreditMemoInvApplicationHistory': incremental_created_date, # new 6/13/2022 # Removed 10/24/25 - 'CreditMemoLine': default, # new - 'CreditMemoLineFeed': default, # new - 'CreditMemoLineHistory': incremental_created_date, # new - 'CreditMemoShare': incremental_last_modified, # new - 'CronJobDetail': default_full, - 'CronTrigger': incremental_created_date, - 'CspTrustedSite': default, # new - 'CustomBrand': incremental_last_modified, - 'CustomBrandAsset': incremental_last_modified, - 'CustomHelpMenuItem': default, - 'CustomHelpMenuSection': default, - 'CustomHttpHeader': default, - 'CustomNotificationType': default, - 'CustomPermission': default, - 'CustomPermissionDependency': default, - 'DandBCompany': default, # new - 'Dashboard': default, - 'DashboardComponent': default_full, - 'DashboardComponentFeed': default, - 'DashboardFeed': default, - 'DataAssessmentFieldMetric': default, # new - 'DataAssessmentMetric': default, # new - 'DataAssessmentValueMetric': default, # new - 'DataUseLegalBasis': default, - 'DataUseLegalBasisHistory': incremental_created_date, - 'DataUseLegalBasisShare': incremental_last_modified, - 'DataUsePurpose': default, - 'DataUsePurposeHistory': incremental_created_date, - 'DataUsePurposeShare': incremental_last_modified, - 'DatacloudCompany': default_full, # new - 'DatacloudContact': default_full, # new - 'DatacloudOwnedEntity': default, # new - 'DatacloudPurchaseUsage': default, # new - 'DeleteEvent': default, - 'DigitalWallet': default, # new - 'Document': default, - 'DocumentAttachmentMap': incremental_created_date, - 'Domain': default, - 'DomainSite': default, - 'DuplicateRecordItem': default, # new - 'DuplicateRecordSet': default, # new - 'DuplicateRule': default, - 'EmailCapture': default, # new - 'EmailDomainFilter': default, # new - 'EmailDomainKey': default, # new - 'EmailMessage': default, - 'EmailMessageRelation': default, - 'EmailRelay': default, # new - 'EmailServicesAddress': default, - 'EmailServicesFunction': default, - 'EmailTemplate': default, - 'EmbeddedServiceDetail': default_full, - 'EmbeddedServiceLabel': default_full, - 'EngagementChannelType': default, - 'EngagementChannelTypeFeed': default, - 'EngagementChannelTypeHistory': incremental_created_date, - 'EngagementChannelTypeShare': incremental_last_modified, - 'EnhancedLetterhead': default, - 'EnhancedLetterheadFeed': default, - 'Entitlement': default, # new - 'EntitlementContact': default, # new - 'EntitlementFeed': default, # new - 'EntitlementHistory': incremental_created_date, # new - 'EntitlementTemplate': default, # new - 'EntityDefinition': incremental_last_modified, - 'EntityMilestone': default, # new - 'EntityMilestoneFeed': default, # new - 'EntityMilestoneHistory': incremental_created_date, # new - 'EntitySubscription': incremental_created_date, - 'Event': default, - 'EventBusSubscriber': default_full, - 'EventFeed': default, - 'EventLogFile': default, # new - 'EventRelation': default, - 'ExpressionFilter': default, - 'ExpressionFilterCriteria': default, - 'ExternalDataSource': default, - 'ExternalDataUserAuth': default, - 'ExternalEvent': default, - 'ExternalEventMapping': default, - 'ExternalEventMappingShare': incremental_last_modified, - 'FeedAttachment': default_full, - 'FeedComment': default, - 'FeedItem': default, - 'FeedPollChoice': incremental_created_date, # new - 'FeedPollVote': incremental_last_modified, # new - 'FeedRevision': default, - 'FieldPermissions': default, - 'FieldSecurityClassification': default, # new - 'FileSearchActivity': default, # new - 'FinanceBalanceSnapshot': default, # new - 'FinanceBalanceSnapshotShare': incremental_last_modified, # new - 'FinanceTransaction': default, # new - 'FinanceTransactionShare': incremental_last_modified, # new - 'FiscalYearSettings': default, - 'FlowDefinitionView': incremental_last_modified, - 'FlowInterview': default, - 'FlowInterviewLog': default, - 'FlowInterviewLogEntry': default, - 'FlowInterviewLogShare': incremental_last_modified, - 'FlowInterviewShare': incremental_last_modified, - 'FlowRecordRelation': default, - 'FlowStageRelation': default, - 'Folder': default, - 'FormulaFunction': default_full, # removed 07/17/25 # re-added 10/24/25 - 'FormulaFunctionAllowedType': default_full, # removed 07/17/25 # re-added 10/24/25 - 'FormulaFunctionCategory': default_full, - 'GrantedByLicense': default, - 'Group': default, - 'GroupMember': default, - 'GtwyProvPaymentMethodType': default, # new - 'Holiday': default, - 'IPAddressRange': default, # new - 'Idea': default, - 'IdentityProviderEventStore': default_full, # new - 'IdentityVerificationEvent': default_full, # new - 'IdpEventLog': default_full, # new - 'IframeWhiteListUrl': default, - 'Image': default, # new - 'ImageFeed': default, # new - 'ImageHistory': incremental_created_date, # new - 'ImageShare': incremental_last_modified, # new - 'Individual': default, - 'IndividualHistory': incremental_created_date, - 'IndividualShare': incremental_last_modified, - 'InstalledMobileApp': default, - 'Invoice': default, # new - 'InvoiceFeed': default, # new - 'InvoiceHistory': incremental_created_date, # new - 'InvoiceLine': default, # new - 'InvoiceLineFeed': default, # new - 'InvoiceLineHistory': incremental_created_date, # new - 'InvoiceShare': incremental_last_modified, # new - 'KnowledgeableUser': default, - 'Lead': default, - 'LeadCleanInfo': default, # new - 'LeadFeed': default, - 'LeadHistory': incremental_created_date, - 'LeadShare': incremental_last_modified, - 'LeadStatus': default, - 'LegalEntity': default, # new - 'LegalEntityFeed': default, # new - 'LegalEntityHistory': incremental_created_date, # new - 'LegalEntityShare': incremental_last_modified, # new - 'LightningExitByPageMetrics': default, # new - 'LightningExperienceTheme': default, - 'LightningOnboardingConfig': default, - 'LightningToggleMetrics': default, # new - 'LightningUriEvent': lightning_uri_event_full, # new - 'LightningUsageByAppTypeMetrics': default, # new - 'LightningUsageByBrowserMetrics': default, # new - 'LightningUsageByFlexiPageMetrics': default, # new - 'LightningUsageByPageMetrics': default, # new - 'ListEmail': default, # new - 'ListEmailIndividualRecipient': default, # new - 'ListEmailRecipientSource': default, # new - 'ListEmailShare': incremental_last_modified, # new - 'ListView': default, - 'ListViewChart': default, - 'ListViewEvent': default_full, # new - 'LiveChatSensitiveDataRule': default, # new - 'Location': default, - 'LocationFeed': default, - 'LocationGroup': default, # new - 'LocationGroupAssignment': default, # new - 'LocationGroupFeed': default, # new - 'LocationGroupHistory': incremental_created_date, # new - 'LocationGroupShare': incremental_last_modified, # new - 'LocationHistory': incremental_created_date, - 'LocationShare': incremental_last_modified, - 'LoginAsEvent': default_full, # new - 'LoginEvent': default_full, # new - 'LoginGeo': default, # new - 'LoginHistory': {self.PRIMARY_KEYS: {'Id'}, self.REPLICATION_KEYS: {'LoginTime'},self.REPLICATION_METHOD: self.INCREMENTAL,}, - 'LoginIp': incremental_created_date, - 'LogoutEvent': default_full, # new + "CreditMemoLine": default, # new + "CreditMemoLineFeed": default, # new + "CreditMemoLineHistory": incremental_created_date, # new + "CreditMemoShare": incremental_last_modified, # new + "CronJobDetail": default_full, + "CronTrigger": incremental_created_date, + "CspTrustedSite": default, # new + "CustomBrand": incremental_last_modified, + "CustomBrandAsset": incremental_last_modified, + "CustomHelpMenuItem": default, + "CustomHelpMenuSection": default, + "CustomHttpHeader": default, + "CustomNotificationType": default, + "CustomPermission": default, + "CustomPermissionDependency": default, + "DandBCompany": default, # new + "Dashboard": default, + "DashboardComponent": default_full, + "DashboardComponentFeed": default, + "DashboardFeed": default, + "DataAssessmentFieldMetric": default, # new + "DataAssessmentMetric": default, # new + "DataAssessmentValueMetric": default, # new + "DataUseLegalBasis": default, + "DataUseLegalBasisHistory": incremental_created_date, + "DataUseLegalBasisShare": incremental_last_modified, + "DataUsePurpose": default, + "DataUsePurposeHistory": incremental_created_date, + "DataUsePurposeShare": incremental_last_modified, + "DatacloudCompany": default_full, # new + "DatacloudContact": default_full, # new + "DatacloudOwnedEntity": default, # new + "DatacloudPurchaseUsage": default, # new + "DeleteEvent": default, + "DigitalWallet": default, # new + "Document": default, + "DocumentAttachmentMap": incremental_created_date, + "Domain": default, + "DomainSite": default, + "DuplicateRecordItem": default, # new + "DuplicateRecordSet": default, # new + "DuplicateRule": default, + "EmailCapture": default, # new + "EmailDomainFilter": default, # new + "EmailDomainKey": default, # new + "EmailMessage": default, + "EmailMessageRelation": default, + "EmailRelay": default, # new + "EmailServicesAddress": default, + "EmailServicesFunction": default, + "EmailTemplate": default, + "EmbeddedServiceDetail": default_full, + "EmbeddedServiceLabel": default_full, + "EngagementChannelType": default, + "EngagementChannelTypeFeed": default, + "EngagementChannelTypeHistory": incremental_created_date, + "EngagementChannelTypeShare": incremental_last_modified, + "EnhancedLetterhead": default, + "EnhancedLetterheadFeed": default, + "Entitlement": default, # new + "EntitlementContact": default, # new + "EntitlementFeed": default, # new + "EntitlementHistory": incremental_created_date, # new + "EntitlementTemplate": default, # new + "EntityDefinition": incremental_last_modified, + "EntityMilestone": default, # new + "EntityMilestoneFeed": default, # new + "EntityMilestoneHistory": incremental_created_date, # new + "EntitySubscription": incremental_created_date, + "Event": default, + "EventBusSubscriber": default_full, + "EventFeed": default, + "EventLogFile": default, # new + "EventRelation": default, + "ExpressionFilter": default, + "ExpressionFilterCriteria": default, + "ExternalDataSource": default, + "ExternalDataUserAuth": default, + "ExternalEvent": default, + "ExternalEventMapping": default, + "ExternalEventMappingShare": incremental_last_modified, + "FeedAttachment": default_full, + "FeedComment": default, + "FeedItem": default, + "FeedPollChoice": incremental_created_date, # new + "FeedPollVote": incremental_last_modified, # new + "FeedRevision": default, + "FieldPermissions": default, + "FieldSecurityClassification": default, # new + "FileSearchActivity": default, # new + "FinanceBalanceSnapshot": default, # new + "FinanceBalanceSnapshotShare": incremental_last_modified, # new + "FinanceTransaction": default, # new + "FinanceTransactionShare": incremental_last_modified, # new + "FiscalYearSettings": default, + "FlowDefinitionView": incremental_last_modified, + "FlowInterview": default, + "FlowInterviewLog": default, + "FlowInterviewLogEntry": default, + "FlowInterviewLogShare": incremental_last_modified, + "FlowInterviewShare": incremental_last_modified, + "FlowRecordRelation": default, + "FlowStageRelation": default, + "Folder": default, + "FormulaFunction": default_full, # removed 07/17/25 # re-added 10/24/25 + "FormulaFunctionAllowedType": default_full, # removed 07/17/25 # re-added 10/24/25 + "FormulaFunctionCategory": default_full, + "GrantedByLicense": default, + "Group": default, + "GroupMember": default, + "GtwyProvPaymentMethodType": default, # new + "Holiday": default, + "IPAddressRange": default, # new + "Idea": default, + "IdentityProviderEventStore": default_full, # new + "IdentityVerificationEvent": default_full, # new + "IdpEventLog": default_full, # new + "IframeWhiteListUrl": default, + "Image": default, # new + "ImageFeed": default, # new + "ImageHistory": incremental_created_date, # new + "ImageShare": incremental_last_modified, # new + "Individual": default, + "IndividualHistory": incremental_created_date, + "IndividualShare": incremental_last_modified, + "InstalledMobileApp": default, + "Invoice": default, # new + "InvoiceFeed": default, # new + "InvoiceHistory": incremental_created_date, # new + "InvoiceLine": default, # new + "InvoiceLineFeed": default, # new + "InvoiceLineHistory": incremental_created_date, # new + "InvoiceShare": incremental_last_modified, # new + "KnowledgeableUser": default, + "Lead": default, + "LeadCleanInfo": default, # new + "LeadFeed": default, + "LeadHistory": incremental_created_date, + "LeadShare": incremental_last_modified, + "LeadStatus": default, + "LegalEntity": default, # new + "LegalEntityFeed": default, # new + "LegalEntityHistory": incremental_created_date, # new + "LegalEntityShare": incremental_last_modified, # new + "LightningExitByPageMetrics": default, # new + "LightningExperienceTheme": default, + "LightningOnboardingConfig": default, + "LightningToggleMetrics": default, # new + "LightningUriEvent": lightning_uri_event_full, # new + "LightningUsageByAppTypeMetrics": default, # new + "LightningUsageByBrowserMetrics": default, # new + "LightningUsageByFlexiPageMetrics": default, # new + "LightningUsageByPageMetrics": default, # new + "ListEmail": default, # new + "ListEmailIndividualRecipient": default, # new + "ListEmailRecipientSource": default, # new + "ListEmailShare": incremental_last_modified, # new + "ListView": default, + "ListViewChart": default, + "ListViewEvent": default_full, # new + "LiveChatSensitiveDataRule": default, # new + "Location": default, + "LocationFeed": default, + "LocationGroup": default, # new + "LocationGroupAssignment": default, # new + "LocationGroupFeed": default, # new + "LocationGroupHistory": incremental_created_date, # new + "LocationGroupShare": incremental_last_modified, # new + "LocationHistory": incremental_created_date, + "LocationShare": incremental_last_modified, + "LoginAsEvent": default_full, # new + "LoginEvent": default_full, # new + "LoginGeo": default, # new + "LoginHistory": { + self.PRIMARY_KEYS: {"Id"}, + self.REPLICATION_KEYS: {"LoginTime"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + }, + "LoginIp": incremental_created_date, + "LogoutEvent": default_full, # new # 'MLField': default, # removed 6/13/2022 added 7/10/2022, removed 06/12/2023 - 'MLPredictionDefinition': default, # removed # 6/13/2022 added back 7/10/2022 - 'Macro': default, - 'MacroHistory': incremental_created_date, - 'MacroInstruction': default, - 'MacroShare': incremental_last_modified, - 'MacroUsage': default, - 'MacroUsageShare': incremental_last_modified, - 'MailmergeTemplate': default, - 'MatchingInformation': default, - 'MatchingRule': default, - 'MatchingRuleItem': default, - 'MessagingChannel': default, # new - 'MessagingChannelSkill': default, # new - 'MessagingConfiguration': default, # new - 'MessagingDeliveryError': default, # new - 'MessagingEndUser': default, # new - 'MessagingEndUserHistory': incremental_created_date, # new - 'MessagingEndUserShare': incremental_last_modified, # new - 'MessagingLink': default, # new - 'MessagingSession': default, # new - 'MessagingSessionFeed': default, # new - 'MessagingSessionHistory': incremental_created_date, # new - 'MessagingSessionShare': incremental_last_modified, # new - 'MessagingTemplate': default, # new - 'MilestoneType': default, # new - 'MobileApplicationDetail': default, - 'MsgChannelLanguageKeyword': default, # new - 'MutingPermissionSet': default, - 'MyDomainDiscoverableLogin': default, - 'NamedCredential': default, - 'Note': default, - 'OauthCustomScope': default, # new - 'OauthCustomScopeApp': default, # new - 'OauthToken': incremental_created_date, - 'ObjectPermissions': default, - 'OnboardingMetrics': default, - 'OperatingHours': default, # new - 'OperatingHoursFeed': default, # new - 'Opportunity': default, - 'OpportunityCompetitor': default, - 'OpportunityContactRole': default, - 'OpportunityFeed': default, - 'OpportunityFieldHistory': incremental_created_date, - 'OpportunityHistory': default, - 'OpportunityLineItem': default, - 'OpportunityPartner': default, - 'OpportunityShare': incremental_last_modified, - 'OpportunityStage': default, - 'Order': default, - 'OrderFeed': default, - 'OrderHistory': incremental_created_date, - 'OrderItem': default, - 'OrderItemFeed': default, - 'OrderItemHistory': incremental_created_date, - 'OrderShare': incremental_last_modified, - 'OrgDeleteRequest': default, # new - 'OrgDeleteRequestShare': incremental_last_modified, # new - 'OrgWideEmailAddress': default, - 'Organization': default, - 'PackageLicense': default, # new - 'Partner': default, - 'PartyConsent': default, - 'PartyConsentFeed': default, - 'PartyConsentHistory': incremental_created_date, - 'PartyConsentShare': incremental_last_modified, - 'Payment': default, # new - 'PaymentAuthAdjustment': default, # new - 'PaymentAuthorization': default, # new - 'PaymentGateway': default, # new - 'PaymentGatewayLog': default, # new - 'PaymentGatewayProvider': default, # new - 'PaymentGroup': default, # new - 'PaymentLineInvoice': default, # new - 'PaymentMethod': default, # new - 'Period': default, - 'PermissionSet': default, - 'PermissionSetAssignment': default, - 'PermissionSetGroup': default, - 'PermissionSetGroupComponent': default, - 'PermissionSetLicense': default, - 'PermissionSetLicenseAssign': default, - 'PermissionSetTabSetting': default, - 'PlatformCachePartition': default, - 'PlatformCachePartitionType': default, - 'PlatformEventUsageMetric': default_full, - 'Pricebook2': default, - 'Pricebook2History': incremental_created_date, - 'PricebookEntry': default, - 'PricebookEntryHistory': incremental_created_date, - 'ProcessDefinition': default, - 'ProcessException': default, # new - 'ProcessExceptionShare': incremental_last_modified, # new - 'ProcessInstance': default, - 'ProcessInstanceNode': default, - 'ProcessInstanceStep': default, - 'ProcessInstanceWorkitem': default, - 'ProcessNode': default, - 'Product2': default, - 'Product2Feed': default, - 'Product2History': incremental_created_date, - 'ProductConsumptionSchedule': default, # new - 'ProductEntitlementTemplate': default, # new - 'Profile': default, - 'Prompt': default, - 'PromptAction': default, # new - 'PromptActionShare': incremental_last_modified, # new - 'PromptError': default, # new - 'PromptErrorShare': incremental_last_modified, # new - 'PromptVersion': default, - 'Publisher': default_full, - 'PushTopic': default, # new - 'QueueSobject': default, - 'QuickText': default, - 'QuickTextHistory': incremental_created_date, - 'QuickTextShare': incremental_last_modified, - 'QuickTextUsage': default, - 'QuickTextUsageShare': incremental_last_modified, - 'Recommendation': default, - 'RecommendationResponse': default, # new 6/13/2022 - 'RecordAction': default, - 'RecordType': default, - 'RedirectWhitelistUrl': default, - 'Refund': default, # new - 'RefundLinePayment': default, # new - 'Report': default, - 'ReportAnomalyEventStore': default, # new - 'ReportAnomalyEventStoreFeed': default, # new - 'ReportEvent': default_full, # new - 'ReportFeed': default, - 'ResourceAbsence': default, # new - 'ResourceAbsenceFeed': default, # new - 'ResourceAbsenceHistory': incremental_created_date, # new - 'ResourcePreference': default, # new - 'ResourcePreferenceFeed': default, # new - 'ResourcePreferenceHistory': incremental_created_date, # new - 'ReturnOrder': default, # new - 'ReturnOrderFeed': default, # new - 'ReturnOrderHistory': incremental_created_date, # new - 'ReturnOrderItemAdjustment': default, # new - 'ReturnOrderItemTax': default, # new - 'ReturnOrderLineItem': default, # new - 'ReturnOrderLineItemFeed': default, # new - 'ReturnOrderLineItemHistory': incremental_created_date, # new - 'ReturnOrderShare': incremental_last_modified, # new - 'SPSamlAttributes': default, # new - 'SamlSsoConfig': default, - 'Scontrol': default, - 'SearchPromotionRule': default, # new - 'SecurityCustomBaseline': default, # new - 'ServiceAppointment': default, # new - 'ServiceAppointmentFeed': default, # new - 'ServiceAppointmentHistory': incremental_created_date, # new - 'ServiceAppointmentShare': incremental_last_modified, # new - 'ServiceAppointmentStatus': default, # new - 'ServiceContract': default, # new - 'ServiceContractFeed': default, # new - 'ServiceContractHistory': incremental_created_date, # new - 'ServiceContractShare': incremental_last_modified, # new - 'ServiceResource': default, # new - 'ServiceResourceFeed': default, # new - 'ServiceResourceHistory': incremental_created_date, # new - 'ServiceResourceShare': incremental_last_modified, # new - 'ServiceResourceSkill': default, # new - 'ServiceResourceSkillFeed': default, # new - 'ServiceResourceSkillHistory': incremental_created_date, # new - 'ServiceSetupProvisioning': default, # new - 'ServiceTerritory': default, # new - 'ServiceTerritoryFeed': default, # new - 'ServiceTerritoryHistory': incremental_created_date, # new - 'ServiceTerritoryMember': default, # new - 'ServiceTerritoryMemberFeed': default, # new - 'ServiceTerritoryMemberHistory': incremental_created_date, # new - 'ServiceTerritoryShare': incremental_last_modified, # new - 'ServiceTerritoryWorkType': default, # new - 'ServiceTerritoryWorkTypeFeed': default, # new - 'ServiceTerritoryWorkTypeHistory': incremental_created_date, # new - 'SessionHijackingEventStore': default, # new - 'SessionHijackingEventStoreFeed': default, # new - 'SessionPermSetActivation': default, - 'SetupAssistantStep': default, # new - 'SetupAuditTrail': incremental_created_date, - 'SetupEntityAccess': default, - 'Site': default, - 'SiteFeed': default, - 'SiteHistory': incremental_created_date, - 'SiteIframeWhiteListUrl': default, - 'SiteRedirectMapping': default, - 'Skill': default, # new - 'SkillRequirement': default, # new - 'SkillRequirementFeed': default, # new - 'SkillRequirementHistory': incremental_created_date, # new - 'SlaProcess': default, # new - 'Solution': default, - 'SolutionFeed': default, - 'SolutionHistory': incremental_created_date, - 'Stamp': default, - 'StampAssignment': default, - 'StaticResource': default, - 'StreamingChannel': default, # new - 'StreamingChannelShare': incremental_last_modified, # new - 'TabDefinition': default_full, - 'Task': default, - 'TaskFeed': default, - 'TenantUsageEntitlement': default, - 'TestSuiteMembership': default, - 'ThirdPartyAccountLink': default_full, - 'ThreatDetectionFeedback': default, # new - 'ThreatDetectionFeedbackFeed': default, # new - 'TimeSlot': default, # new - 'TodayGoal': default, - 'TodayGoalShare': incremental_last_modified, - 'Topic': default, - 'TopicAssignment': default, - 'TopicFeed': default, - 'TopicUserEvent': incremental_created_date, # new - 'TransactionSecurityPolicy': default, # new - 'Translation': default, - 'UiFormulaCriterion': default, - 'UiFormulaRule': default, - 'UriEvent': default_full, # new - 'User': default, - 'UserAppInfo': default, - 'UserAppMenuCustomization': default, - 'UserAppMenuCustomizationShare': incremental_last_modified, - 'UserAppMenuItem': default_full, - 'UserEmailPreferredPerson': default, - 'UserEmailPreferredPersonShare': incremental_last_modified, - 'UserFeed': default, - 'UserLicense': default, - 'UserListView': default, - 'UserListViewCriterion': default, - 'UserLogin': incremental_last_modified, - 'UserPackageLicense': default, # new - 'UserPermissionAccess': default_full, - 'UserPreference': default, - 'UserProvAccount': default, # new - 'UserProvAccountStaging': default, # new - 'UserProvMockTarget': default, # new - 'UserProvisioningConfig': default, # new - 'UserProvisioningLog': default, # new - 'UserProvisioningRequest': default, # new - 'UserProvisioningRequestShare': incremental_last_modified, # new - 'UserRole': default, - 'UserSetupEntityAccess': default_full, - 'UserShare': incremental_last_modified, - 'VerificationHistory': default, # new - 'VisualforceAccessMetrics': default, # new - 'WaveAutoInstallRequest': default, - 'WaveCompatibilityCheckItem': default, - 'WebLink': default, # new - 'WorkOrder': default, # new - 'WorkOrderFeed': default, # new - 'WorkOrderHistory': incremental_created_date, # new - 'WorkOrderLineItem': default, # new - 'WorkOrderLineItemFeed': default, # new - 'WorkOrderLineItemHistory': incremental_created_date, # new - 'WorkOrderLineItemStatus': default, # new - 'WorkOrderShare': incremental_last_modified, # new - 'WorkOrderStatus': default, # new - 'WorkType': default, # new - 'WorkTypeFeed': default, # new - 'WorkTypeGroup': default, # new - 'WorkTypeGroupFeed': default, # new - 'WorkTypeGroupHistory': incremental_created_date, # new - 'WorkTypeGroupMember': default, # new - 'WorkTypeGroupMemberFeed': default, # new - 'WorkTypeGroupMemberHistory': incremental_created_date, # new - 'WorkTypeGroupShare': incremental_last_modified, # new - 'WorkTypeHistory': incremental_created_date, # new - 'WorkTypeShare': incremental_last_modified, # new - 'RecentlyViewed': default_full, # REST ONLY STREAM - 'TaskPriority': default, # REST ONLY STREAM - 'DeclinedEventRelation': default, # REST ONLY STREAM - 'AcceptedEventRelation': default, # REST ONLY STREAM - 'OrderStatus': default, # REST ONLY STREAM - 'SolutionStatus': default, # REST ONLY STREAM - 'CaseStatus': default, # REST ONLY STREAM - 'TaskStatus': default, # REST ONLY STREAM - 'PartnerRole': default, # REST ONLY STREAM - 'ContractStatus': default, # REST ONLY STREAM - 'UndecidedEventRelation': default, # REST ONLY STREAM + "MLPredictionDefinition": default, # removed # 6/13/2022 added back 7/10/2022 + "Macro": default, + "MacroHistory": incremental_created_date, + "MacroInstruction": default, + "MacroShare": incremental_last_modified, + "MacroUsage": default, + "MacroUsageShare": incremental_last_modified, + "MailmergeTemplate": default, + "MatchingInformation": default, + "MatchingRule": default, + "MatchingRuleItem": default, + "MessagingChannel": default, # new + "MessagingChannelSkill": default, # new + "MessagingConfiguration": default, # new + "MessagingDeliveryError": default, # new + "MessagingEndUser": default, # new + "MessagingEndUserHistory": incremental_created_date, # new + "MessagingEndUserShare": incremental_last_modified, # new + "MessagingLink": default, # new + "MessagingSession": default, # new + "MessagingSessionFeed": default, # new + "MessagingSessionHistory": incremental_created_date, # new + "MessagingSessionShare": incremental_last_modified, # new + "MessagingTemplate": default, # new + "MilestoneType": default, # new + "MobileApplicationDetail": default, + "MsgChannelLanguageKeyword": default, # new + "MutingPermissionSet": default, + "MyDomainDiscoverableLogin": default, + "NamedCredential": default, + "Note": default, + "OauthCustomScope": default, # new + "OauthCustomScopeApp": default, # new + "OauthToken": incremental_created_date, + "ObjectPermissions": default, + "OnboardingMetrics": default, + "OperatingHours": default, # new + "OperatingHoursFeed": default, # new + "Opportunity": default, + "OpportunityCompetitor": default, + "OpportunityContactRole": default, + "OpportunityFeed": default, + "OpportunityFieldHistory": incremental_created_date, + "OpportunityHistory": default, + "OpportunityLineItem": default, + "OpportunityPartner": default, + "OpportunityShare": incremental_last_modified, + "OpportunityStage": default, + "Order": default, + "OrderFeed": default, + "OrderHistory": incremental_created_date, + "OrderItem": default, + "OrderItemFeed": default, + "OrderItemHistory": incremental_created_date, + "OrderShare": incremental_last_modified, + "OrgDeleteRequest": default, # new + "OrgDeleteRequestShare": incremental_last_modified, # new + "OrgWideEmailAddress": default, + "Organization": default, + "PackageLicense": default, # new + "Partner": default, + "PartyConsent": default, + "PartyConsentFeed": default, + "PartyConsentHistory": incremental_created_date, + "PartyConsentShare": incremental_last_modified, + "Payment": default, # new + "PaymentAuthAdjustment": default, # new + "PaymentAuthorization": default, # new + "PaymentGateway": default, # new + "PaymentGatewayLog": default, # new + "PaymentGatewayProvider": default, # new + "PaymentGroup": default, # new + "PaymentLineInvoice": default, # new + "PaymentMethod": default, # new + "Period": default, + "PermissionSet": default, + "PermissionSetAssignment": default, + "PermissionSetGroup": default, + "PermissionSetGroupComponent": default, + "PermissionSetLicense": default, + "PermissionSetLicenseAssign": default, + "PermissionSetTabSetting": default, + "PlatformCachePartition": default, + "PlatformCachePartitionType": default, + "PlatformEventUsageMetric": default_full, + "Pricebook2": default, + "Pricebook2History": incremental_created_date, + "PricebookEntry": default, + "PricebookEntryHistory": incremental_created_date, + "ProcessDefinition": default, + "ProcessException": default, # new + "ProcessExceptionShare": incremental_last_modified, # new + "ProcessInstance": default, + "ProcessInstanceNode": default, + "ProcessInstanceStep": default, + "ProcessInstanceWorkitem": default, + "ProcessNode": default, + "Product2": default, + "Product2Feed": default, + "Product2History": incremental_created_date, + "ProductConsumptionSchedule": default, # new + "ProductEntitlementTemplate": default, # new + "Profile": default, + "Prompt": default, + "PromptAction": default, # new + "PromptActionShare": incremental_last_modified, # new + "PromptError": default, # new + "PromptErrorShare": incremental_last_modified, # new + "PromptVersion": default, + "Publisher": default_full, + "PushTopic": default, # new + "QueueSobject": default, + "QuickText": default, + "QuickTextHistory": incremental_created_date, + "QuickTextShare": incremental_last_modified, + "QuickTextUsage": default, + "QuickTextUsageShare": incremental_last_modified, + "Recommendation": default, + "RecommendationResponse": default, # new 6/13/2022 + "RecordAction": default, + "RecordType": default, + "RedirectWhitelistUrl": default, + "Refund": default, # new + "RefundLinePayment": default, # new + "Report": default, + "ReportAnomalyEventStore": default, # new + "ReportAnomalyEventStoreFeed": default, # new + "ReportEvent": default_full, # new + "ReportFeed": default, + "ResourceAbsence": default, # new + "ResourceAbsenceFeed": default, # new + "ResourceAbsenceHistory": incremental_created_date, # new + "ResourcePreference": default, # new + "ResourcePreferenceFeed": default, # new + "ResourcePreferenceHistory": incremental_created_date, # new + "ReturnOrder": default, # new + "ReturnOrderFeed": default, # new + "ReturnOrderHistory": incremental_created_date, # new + "ReturnOrderItemAdjustment": default, # new + "ReturnOrderItemTax": default, # new + "ReturnOrderLineItem": default, # new + "ReturnOrderLineItemFeed": default, # new + "ReturnOrderLineItemHistory": incremental_created_date, # new + "ReturnOrderShare": incremental_last_modified, # new + "SPSamlAttributes": default, # new + "SamlSsoConfig": default, + "Scontrol": default, + "SearchPromotionRule": default, # new + "SecurityCustomBaseline": default, # new + "ServiceAppointment": default, # new + "ServiceAppointmentFeed": default, # new + "ServiceAppointmentHistory": incremental_created_date, # new + "ServiceAppointmentShare": incremental_last_modified, # new + "ServiceAppointmentStatus": default, # new + "ServiceContract": default, # new + "ServiceContractFeed": default, # new + "ServiceContractHistory": incremental_created_date, # new + "ServiceContractShare": incremental_last_modified, # new + "ServiceResource": default, # new + "ServiceResourceFeed": default, # new + "ServiceResourceHistory": incremental_created_date, # new + "ServiceResourceShare": incremental_last_modified, # new + "ServiceResourceSkill": default, # new + "ServiceResourceSkillFeed": default, # new + "ServiceResourceSkillHistory": incremental_created_date, # new + "ServiceSetupProvisioning": default, # new + "ServiceTerritory": default, # new + "ServiceTerritoryFeed": default, # new + "ServiceTerritoryHistory": incremental_created_date, # new + "ServiceTerritoryMember": default, # new + "ServiceTerritoryMemberFeed": default, # new + "ServiceTerritoryMemberHistory": incremental_created_date, # new + "ServiceTerritoryShare": incremental_last_modified, # new + "ServiceTerritoryWorkType": default, # new + "ServiceTerritoryWorkTypeFeed": default, # new + "ServiceTerritoryWorkTypeHistory": incremental_created_date, # new + "SessionHijackingEventStore": default, # new + "SessionHijackingEventStoreFeed": default, # new + "SessionPermSetActivation": default, + "SetupAssistantStep": default, # new + "SetupAuditTrail": incremental_created_date, + "SetupEntityAccess": default, + "Site": default, + "SiteFeed": default, + "SiteHistory": incremental_created_date, + "SiteIframeWhiteListUrl": default, + "SiteRedirectMapping": default, + "Skill": default, # new + "SkillRequirement": default, # new + "SkillRequirementFeed": default, # new + "SkillRequirementHistory": incremental_created_date, # new + "SlaProcess": default, # new + "Solution": default, + "SolutionFeed": default, + "SolutionHistory": incremental_created_date, + "Stamp": default, + "StampAssignment": default, + "StaticResource": default, + "StreamingChannel": default, # new + "StreamingChannelShare": incremental_last_modified, # new + "TabDefinition": default_full, + "Task": default, + "TaskFeed": default, + "TenantUsageEntitlement": default, + "TestSuiteMembership": default, + "ThirdPartyAccountLink": default_full, + "ThreatDetectionFeedback": default, # new + "ThreatDetectionFeedbackFeed": default, # new + "TimeSlot": default, # new + "TodayGoal": default, + "TodayGoalShare": incremental_last_modified, + "Topic": default, + "TopicAssignment": default, + "TopicFeed": default, + "TopicUserEvent": incremental_created_date, # new + "TransactionSecurityPolicy": default, # new + "Translation": default, + "UiFormulaCriterion": default, + "UiFormulaRule": default, + "UriEvent": default_full, # new + "User": default, + "UserAppInfo": default, + "UserAppMenuCustomization": default, + "UserAppMenuCustomizationShare": incremental_last_modified, + "UserAppMenuItem": default_full, + "UserEmailPreferredPerson": default, + "UserEmailPreferredPersonShare": incremental_last_modified, + "UserFeed": default, + "UserLicense": default, + "UserListView": default, + "UserListViewCriterion": default, + "UserLogin": incremental_last_modified, + "UserPackageLicense": default, # new + "UserPermissionAccess": default_full, + "UserPreference": default, + "UserProvAccount": default, # new + "UserProvAccountStaging": default, # new + "UserProvMockTarget": default, # new + "UserProvisioningConfig": default, # new + "UserProvisioningLog": default, # new + "UserProvisioningRequest": default, # new + "UserProvisioningRequestShare": incremental_last_modified, # new + "UserRole": default, + "UserSetupEntityAccess": default_full, + "UserShare": incremental_last_modified, + "VerificationHistory": default, # new + "VisualforceAccessMetrics": default, # new + "WaveAutoInstallRequest": default, + "WaveCompatibilityCheckItem": default, + "WebLink": default, # new + "WorkOrder": default, # new + "WorkOrderFeed": default, # new + "WorkOrderHistory": incremental_created_date, # new + "WorkOrderLineItem": default, # new + "WorkOrderLineItemFeed": default, # new + "WorkOrderLineItemHistory": incremental_created_date, # new + "WorkOrderLineItemStatus": default, # new + "WorkOrderShare": incremental_last_modified, # new + "WorkOrderStatus": default, # new + "WorkType": default, # new + "WorkTypeFeed": default, # new + "WorkTypeGroup": default, # new + "WorkTypeGroupFeed": default, # new + "WorkTypeGroupHistory": incremental_created_date, # new + "WorkTypeGroupMember": default, # new + "WorkTypeGroupMemberFeed": default, # new + "WorkTypeGroupMemberHistory": incremental_created_date, # new + "WorkTypeGroupShare": incremental_last_modified, # new + "WorkTypeHistory": incremental_created_date, # new + "WorkTypeShare": incremental_last_modified, # new + "RecentlyViewed": default_full, # REST ONLY STREAM + "TaskPriority": default, # REST ONLY STREAM + "DeclinedEventRelation": default, # REST ONLY STREAM + "AcceptedEventRelation": default, # REST ONLY STREAM + "OrderStatus": default, # REST ONLY STREAM + "SolutionStatus": default, # REST ONLY STREAM + "CaseStatus": default, # REST ONLY STREAM + "TaskStatus": default, # REST ONLY STREAM + "PartnerRole": default, # REST ONLY STREAM + "ContractStatus": default, # REST ONLY STREAM + "UndecidedEventRelation": default, # REST ONLY STREAM # Newly discovered as of 2/12/2022 - 'BriefcaseAssignment': default, - 'BriefcaseDefinition': default, - 'BriefcaseRule': default, - 'BriefcaseRuleFilter': default, - 'CartCheckoutSession': default, # removed # 6/13/2022 - 'CartDeliveryGroup': default, # removed # 6/13/2022 - 'CartItem': default, # removed # 6/13/2022 - 'CartItemPriceAdjustment': default, # added # 10/19/2022 - 'CartRelatedItem': default, # removed # 6/13/2022 - 'CartTax': default, # removed # 6/13/2022 - 'CartValidationOutput': default, # removed # 6/13/2022 - 'Conversation': incremental_last_modified, # added # 4/11/2023 - 'ConversationParticipant': incremental_last_modified, # added # 4/11/2023 - 'Coupon': default, # added # 10/18/2022 - 'CouponHistory': incremental_created_date, # added # 10/18/2022 - 'CouponShare': incremental_last_modified, # added # 10/18/2022 - 'OperatingHoursHoliday': default, - 'OperatingHoursHolidayFeed': default, + "BriefcaseAssignment": default, + "BriefcaseDefinition": default, + "BriefcaseRule": default, + "BriefcaseRuleFilter": default, + "CartCheckoutSession": default, # removed # 6/13/2022 + "CartDeliveryGroup": default, # removed # 6/13/2022 + "CartItem": default, # removed # 6/13/2022 + "CartItemPriceAdjustment": default, # added # 10/19/2022 + "CartRelatedItem": default, # removed # 6/13/2022 + "CartTax": default, # removed # 6/13/2022 + "CartValidationOutput": default, # removed # 6/13/2022 + "Conversation": incremental_last_modified, # added # 4/11/2023 + "ConversationParticipant": incremental_last_modified, # added # 4/11/2023 + "Coupon": default, # added # 10/18/2022 + "CouponHistory": incremental_created_date, # added # 10/18/2022 + "CouponShare": incremental_last_modified, # added # 10/18/2022 + "OperatingHoursHoliday": default, + "OperatingHoursHolidayFeed": default, #'PaymentFeed': default, # added # 10/18/2022 # removed 01/13/23 - 'PermissionSetEventStore': default_full, - 'ProductAttribute': default, # added # 10/18/2022 - 'ProductAttributeSet': default, # added # 10/18/2022 - 'ProductAttributeSetItem': default, # added # 10/18/2022 - 'ProductAttributeSetProduct': default, # added # 10/18/2022 - 'Promotion': default, # added # 10/18/2022 - 'PromotionFeed': default, # added # 10/18/2022 - 'PromotionHistory': incremental_created_date, # added # 10/18/2022 - 'PromotionSegment': default, # added # 10/18/2022 - 'PromotionMarketSegment': default, # added # 10/18/2022 - 'PromotionMarketSegmentFeed': default, # added # 10/18/2022 - 'PromotionMarketSegmentHistory': incremental_created_date, # added # 10/18/2022 - 'PromotionSegmentHistory': incremental_created_date, # added # 10/18/2022 - 'PromotionSegmentSalesStore': default, # added # 10/18/2022 - 'PromotionSegmentSalesStoreHistory': incremental_created_date, # added # 10/18/2022 - 'PromotionSegmentShare': incremental_last_modified, # added # 10/18/2022 - 'PromotionShare': incremental_last_modified, # added # 10/18/2022 - 'PromotionTarget': default, # added # 10/18/2022 - 'PromotionTargetHistory': incremental_created_date, # added # 10/18/2022 - 'PromotionQualifier': default, # added # 10/18/2022 - 'PromotionQualifierHistory': incremental_created_date, # added # 10/18/2022 - 'SalesStore': default_full, # added # 02/15/2023 - 'Shift': default, - 'ShiftFeed': default, - 'ShiftHistory': incremental_created_date, - 'ShiftShare': incremental_last_modified, - 'ShiftStatus': default, - 'TapTester__c': default, # added 8/4/2023 - 'TapTester__Share': incremental_last_modified, # added 8/4/2023 - 'WebCart': default, # re-added # 10/18/2022 - 'WebCartAdjustmentGroup': default, # added # 10/18/2022 - 'WebCartHistory': incremental_created_date, # re-added # 10/18/2022 - 'WebCartShare': incremental_last_modified, # re-added # 10/18/2022 - 'WebStore': default, # re-added # 10/18/2022 - 'WebStoreShare': incremental_last_modified, # re-added # 10/18/2022 - 'WorkPlan': default, - 'WorkPlanFeed': default, - 'WorkPlanHistory': incremental_created_date, - 'WorkPlanShare': incremental_last_modified, - 'WorkPlanTemplate': default, - 'WorkPlanTemplateEntry': default, - 'WorkPlanTemplateEntryFeed': default, - 'WorkPlanTemplateEntryHistory': incremental_created_date, - 'WorkPlanTemplateFeed': default, - 'WorkPlanTemplateHistory': incremental_created_date, - 'WorkPlanTemplateShare': incremental_last_modified, - 'WorkStep': default, - 'WorkStepFeed': default, - 'WorkStepHistory': incremental_created_date, - 'WorkStepStatus': default, - 'WorkStepTemplate': default, - 'WorkStepTemplateFeed': default, - 'WorkStepTemplateHistory': incremental_created_date, - 'WorkStepTemplateShare': incremental_last_modified, - #added on 10/14/2023 - 'BuyerGroup': default, - 'BuyerGroupFeed': default, - 'BuyerGroupHistory': incremental_created_date, - 'BuyerGroupShare': incremental_last_modified, - 'CartDeliveryGroupMethod': default, - 'ProductCatalog': default, - 'ProductCatalogFeed': default, - 'ProductCatalogHistory': incremental_created_date, - 'ProductCatalogShare': incremental_last_modified, - 'ProductCategory': default, - 'ProductCategoryFeed': default, - 'ProductCategoryHistory': incremental_created_date, - 'ProductCategoryProduct': default, - 'ProductCategoryProductHistory': incremental_created_date, - 'PromotionSegmentBuyerGroup': default, - 'PromotionSegmentBuyerGroupHistory': incremental_created_date, - 'WebStoreBuyerGroup': default, - 'WebStoreCatalog': default, - 'WebStoreCatalogHistory': incremental_created_date, + "PermissionSetEventStore": default_full, + "ProductAttribute": default, # added # 10/18/2022 + "ProductAttributeSet": default, # added # 10/18/2022 + "ProductAttributeSetItem": default, # added # 10/18/2022 + "ProductAttributeSetProduct": default, # added # 10/18/2022 + "Promotion": default, # added # 10/18/2022 + "PromotionFeed": default, # added # 10/18/2022 + "PromotionHistory": incremental_created_date, # added # 10/18/2022 + "PromotionSegment": default, # added # 10/18/2022 + "PromotionMarketSegment": default, # added # 10/18/2022 + "PromotionMarketSegmentFeed": default, # added # 10/18/2022 + "PromotionMarketSegmentHistory": incremental_created_date, # added # 10/18/2022 + "PromotionSegmentHistory": incremental_created_date, # added # 10/18/2022 + "PromotionSegmentSalesStore": default, # added # 10/18/2022 + "PromotionSegmentSalesStoreHistory": incremental_created_date, # added # 10/18/2022 + "PromotionSegmentShare": incremental_last_modified, # added # 10/18/2022 + "PromotionShare": incremental_last_modified, # added # 10/18/2022 + "PromotionTarget": default, # added # 10/18/2022 + "PromotionTargetHistory": incremental_created_date, # added # 10/18/2022 + "PromotionQualifier": default, # added # 10/18/2022 + "PromotionQualifierHistory": incremental_created_date, # added # 10/18/2022 + "SalesStore": default_full, # added # 02/15/2023 + "Shift": default, + "ShiftFeed": default, + "ShiftHistory": incremental_created_date, + "ShiftShare": incremental_last_modified, + "ShiftStatus": default, + "TapTester__c": default, # added 8/4/2023 + "TapTester__Share": incremental_last_modified, # added 8/4/2023 + "WebCart": default, # re-added # 10/18/2022 + "WebCartAdjustmentGroup": default, # added # 10/18/2022 + "WebCartHistory": incremental_created_date, # re-added # 10/18/2022 + "WebCartShare": incremental_last_modified, # re-added # 10/18/2022 + "WebStore": default, # re-added # 10/18/2022 + "WebStoreShare": incremental_last_modified, # re-added # 10/18/2022 + "WorkPlan": default, + "WorkPlanFeed": default, + "WorkPlanHistory": incremental_created_date, + "WorkPlanShare": incremental_last_modified, + "WorkPlanTemplate": default, + "WorkPlanTemplateEntry": default, + "WorkPlanTemplateEntryFeed": default, + "WorkPlanTemplateEntryHistory": incremental_created_date, + "WorkPlanTemplateFeed": default, + "WorkPlanTemplateHistory": incremental_created_date, + "WorkPlanTemplateShare": incremental_last_modified, + "WorkStep": default, + "WorkStepFeed": default, + "WorkStepHistory": incremental_created_date, + "WorkStepStatus": default, + "WorkStepTemplate": default, + "WorkStepTemplateFeed": default, + "WorkStepTemplateHistory": incremental_created_date, + "WorkStepTemplateShare": incremental_last_modified, + # added on 10/14/2023 + "BuyerGroup": default, + "BuyerGroupFeed": default, + "BuyerGroupHistory": incremental_created_date, + "BuyerGroupShare": incremental_last_modified, + "CartDeliveryGroupMethod": default, + "ProductCatalog": default, + "ProductCatalogFeed": default, + "ProductCatalogHistory": incremental_created_date, + "ProductCatalogShare": incremental_last_modified, + "ProductCategory": default, + "ProductCategoryFeed": default, + "ProductCategoryHistory": incremental_created_date, + "ProductCategoryProduct": default, + "ProductCategoryProductHistory": incremental_created_date, + "PromotionSegmentBuyerGroup": default, + "PromotionSegmentBuyerGroupHistory": incremental_created_date, + "WebStoreBuyerGroup": default, + "WebStoreCatalog": default, + "WebStoreCatalogHistory": incremental_created_date, # added on 2024/02/19 - 'Address': default, - 'FulfillmentOrderShare': incremental_last_modified, - 'FulfillmentOrderLineItemFeed': default, - 'FulfillmentOrderItemTax': default, - 'FulfillmentOrder': default, - 'FulfillmentOrderItemTaxFeed': default, - 'FulfillmentOrderLineItem': default, - 'FulfillmentOrderItemAdjustment': default, - 'FulfillmentOrderItemAdjustmentFeed': default, - 'FulfillmentOrderFeed': default, - 'OperatingHoursShare': incremental_last_modified, - 'Shipment': default, - 'ShipmentItemHistory': incremental_created_date, - 'ShipmentHistory': incremental_created_date, - 'ShipmentShare': incremental_last_modified, - 'ShipmentItem': default, - 'ShipmentItemFeed': default, - 'ShipmentFeed': default, + "Address": default, + "FulfillmentOrderShare": incremental_last_modified, + "FulfillmentOrderLineItemFeed": default, + "FulfillmentOrderItemTax": default, + "FulfillmentOrder": default, + "FulfillmentOrderItemTaxFeed": default, + "FulfillmentOrderLineItem": default, + "FulfillmentOrderItemAdjustment": default, + "FulfillmentOrderItemAdjustmentFeed": default, + "FulfillmentOrderFeed": default, + "OperatingHoursShare": incremental_last_modified, + "Shipment": default, + "ShipmentItemHistory": incremental_created_date, + "ShipmentHistory": incremental_created_date, + "ShipmentShare": incremental_last_modified, + "ShipmentItem": default, + "ShipmentItemFeed": default, + "ShipmentFeed": default, # added on 2024/10/03 - 'FlowTestResultShare': incremental_last_modified, - 'ProblemShare': incremental_last_modified, - 'FlowOrchestrationStageInstanceShare': incremental_last_modified, - 'ConvMessageSendRequest': default, - 'AppointmentInvitationFeed': default, - 'EngagementChannelWorkType': default, - 'PromotionLineItemRule': default, - 'ShippingCarrier': default, - 'MLModelMetric': default, - 'PromotionTierHistory': incremental_created_date, - 'IncidentRelatedItemHistory': incremental_created_date, - 'ShippingRateGroup': default, - 'ExternalDataSrcDescriptor': default, - 'WaitlistWorkType': default, - 'PrivacyJobSessionShare': incremental_last_modified, - 'ReplyEmailSettings': default, - 'ProblemRelatedItemHistory': incremental_created_date, - 'AppointmentInvitationHistory': incremental_created_date, - 'FlowRecordVersionOccurrence': default, - 'ChangeRequest': default, - 'MsgChannelUsageExternalOrg': default, - 'ManagedContentVariant': default, - 'FlowOrchestrationStepInstance': default, - 'SellerShare': incremental_last_modified, - 'PrivacyJobSession': default, - 'WaitlistServiceResource': default, - 'FlowOrchestrationStepInstanceShare': incremental_last_modified, - 'FlowRecordVersion': default, - 'ScorecardMetric': default, - 'ChangeRequestRelatedItemFeed': default, - 'UserDefinedLabelAssignmentShare': incremental_last_modified, - 'ProblemRelatedItem': default, - 'GuestUserAnomalyEventStoreFeed': default, - 'EngagementChannelWorkTypeFeed': default, - 'SkillType': default, - 'InventoryReservationShare': incremental_last_modified, - 'Problem': default, - 'AppointmentInvitationShare': incremental_last_modified, - 'ChatRetirementRdyMetrics': default, - 'BusinessBrandShare': incremental_last_modified, - 'SellerHistory': incremental_created_date, - 'AppointmentCategoryFeed': default, - 'ProblemIncidentFeed': default, - 'ShippingConfigurationSet': default, - 'MLModel': default, - 'WaitlistWorkTypeHistory': incremental_created_date, - 'ShippingCarrierFeed': default, - 'PromotionSegmentBuyerGroupFeed': default, - 'PrivacySessionRecordFailure': default, - 'ContractLineItemFeed': default, - 'ContextParamMap': default, - 'FlowOrchestrationInstance': default, - 'ActivityFieldHistory': default_full, - 'EngagementChannelWorkTypeHistory': incremental_created_date, - 'WaitlistParticipantHistory': incremental_created_date, - 'WebCartAdjustmentBasis': default, - 'ScorecardAssociation': default, - 'ShippingConfigurationSetShare': incremental_last_modified, - 'PromotionQualifierFeed': default, - 'UserPrioritizedRecordShare': incremental_last_modified, - 'CustomerShare': incremental_last_modified, - 'WaitlistParticipant': default, - 'BusinessBrand': default, - 'PromotionLineItemRuleShare': incremental_last_modified, - 'PrivacyObjectSession': default, - 'FlowRecord': default, - 'ShippingCarrierMethodHistory': incremental_created_date, - 'PrivacyRTBFRequestHistory': incremental_created_date, - 'TableauHostMapping': default, - 'AppointmentInvitation': default, - 'PipelineInspectionListView': default, - 'PromotionTier': default, - 'ServiceAppointmentAttendeeHistory': incremental_created_date, - 'ProcessFlowMigration': default, - 'ServiceAppointmentAttendeeShare': incremental_last_modified, - 'Scorecard': default, - 'ManagedContent': default, - 'ChangeRequestRelatedItem': default, - 'ShiftEngagementChannelFeed': default, - 'EventRelayFeedback': default, - 'CaseRelatedIssueFeed': default, - 'ChangeRequestRelatedIssueHistory': incremental_created_date, - 'Incident': default, - 'EngmtChannelTypeSettings': default, - 'PrivacyRTBFRequestShare': incremental_last_modified, - 'IncidentRelatedItemFeed': default, - 'ProblemHistory': incremental_created_date, - 'ChangeRequestRelatedIssueFeed': default, - 'CouponFeed': default, - 'WaitlistWorkTypeFeed': default, - 'GuestUserAnomalyEventStore': default, - 'AppointmentCategoryHistory': incremental_created_date, - 'OauthTokenExchHandlerApp': default, - 'FlowOrchestrationWorkItem': default, - 'CouponCodeRedemption': default, - 'PromotionTierFeed': default, - 'ExpressionSetView': incremental_last_modified, - 'OpportunityRelatedDeleteLog': default, - 'ShippingCarrierHistory': incremental_created_date, - 'ShippingCarrierMethodFeed': default, - 'ChangeRequestRelatedIssue': default, - 'WaitlistShare': incremental_last_modified, - 'Customer': default, - 'EventRelayConfig': default, - 'ShippingCarrierMethodShare': incremental_last_modified, - 'PrivacyPolicyDefinition': default, - 'ProblemFeed': default, - 'FlowRecordElement': default, - 'ProblemRelatedItemFeed': default, - 'AssignedResourceHistory': incremental_created_date, - 'UserPrioritizedRecord': default, - 'CaseRelatedIssueHistory': incremental_created_date, - 'FlowOrchestrationInstanceShare': incremental_last_modified, - 'FlowOrchestrationLog': default, - 'OauthTokenExchangeHandler': default, - 'BrowserPolicyViolation': default, - 'WaitlistServiceResourceHistory': incremental_created_date, - 'WaitlistHistory': incremental_created_date, - 'WaitlistServiceResourceFeed': default, - 'ShippingRateArea': default, - 'Seller': default, - 'ServiceAppointmentAttendee': default, - 'PrivacyRTBFRequestFeed': default, - 'WaitlistParticipantFeed': default, - 'ChangeRequestShare': incremental_last_modified, - 'ChangeRequestHistory': incremental_created_date, - 'CaseRelatedIssue': default, - 'PendingOrderSummary': default_full, - 'Waitlist': default, - 'CaseHistory2': default, - 'AppointmentInvitee': default, - 'Participant': default, - 'UserDefinedLabelAssignment': default, - 'ManagedContentSpace': default, - 'ProblemIncidentHistory': incremental_created_date, - 'PromotionSegmentFeed': default, - 'ManagedContentChannel': default, - 'ShippingCarrierMethod': default, - 'PrivacySessionRecordFailureShare': incremental_last_modified, - 'InventoryReservation': default, - 'PrivacyRTBFRequest': default, - 'UserDefinedLabelShare': incremental_last_modified, - 'FileEventStore': default_full, - 'WebStoreInventorySource': default, - 'FlowTestResult': default, - 'IncidentRelatedItem': default, - 'PromotionSegmentSalesStoreFeed': default, - 'ScorecardShare': incremental_last_modified, - 'WaitlistFeed': default, - 'PrivacyObjectSessionShare': incremental_last_modified, - 'ShiftWorkTopicHistory': incremental_created_date, - 'CouponCodeRedemptionShare': incremental_last_modified, - 'PrivacyPolicy': default, - 'ShiftEngagementChannelHistory': incremental_created_date, - 'IncidentHistory': incremental_created_date, - 'OrgEmailAddressSecurity': default, - 'ShippingCarrierShare': incremental_last_modified, - 'UserDefinedLabel': default, - 'MessagingChannelUsage': default, - 'IncidentFeed': default, - 'TableauHostMappingShare': incremental_last_modified, - 'ShiftWorkTopic': default, - 'ChangeRequestRelatedItemHistory': incremental_created_date, - 'MLRecommendationDefinition': default, - 'PaymentFeed': default, - 'MLModelFactorComponent': default, - 'ChangeRequestFeed': default, - 'MLModelFactor': default, - 'InventoryItemReservation': default, - 'PromotionTargetFeed': default, - 'ShiftEngagementChannel': default, - 'ProblemIncident': default, - 'ShiftWorkTopicFeed': default, - 'FlowRecordShare': incremental_last_modified, - 'IncidentShare': incremental_last_modified, - 'AppointmentCategory': default, - 'StandardShippingRate': default, - 'MlFeatureValueMetric': default, - 'FlowOrchestrationWorkItemShare': incremental_last_modified, - 'DataWeaveResource': default, - 'ExternalEncryptionRootKey': default_full, - 'ServiceAppointmentAttendeeFeed': default, - 'FlowOrchestrationStageInstance': default, - 'ExtlClntAppOauthPlcyAttr': default, - 'LocationShippingCarrierMethodFeed': default, - 'ExtlClntAppOauthPlcyAttr': default, - 'ExtlClntAppOauthIpRange': default, - 'ExtlClntAppOauthConsumer': default, - 'ExtlClntAppPlcyCnfg': default, - 'DeliveryEstimationSetupShare': incremental_last_modified, - 'ConvIntelligenceSignalSubRule': default, - 'ConvIntelligenceSignalRule': default, - 'ExtlClntAppSampleSettings': default, - 'ExtlClntAppOauthPlcyCnfg': default, - 'ExtlClntAppOauthSettings': default, - 'ExtlClntAppOauthSetCustmScp': default, - 'DeliveryEstimationSetup': default, - 'LocationShippingCarrierMethodShare': incremental_last_modified, - 'ExternalClientApplication': default, - 'LocationShippingCarrierMethodHistory': incremental_created_date, - 'ExtlClntAppSamplePlcyCnfg': default, - 'DeliveryEstimationSetupFeed': default, - 'ExtlClntAppOauthPlcyCustmScp': default, - 'ExtlClntAppOauthSetAttr': default, - 'LocationShippingCarrierMethod': default, - 'DeliveryEstimationSetupHistory': incremental_created_date, + "FlowTestResultShare": incremental_last_modified, + "ProblemShare": incremental_last_modified, + "FlowOrchestrationStageInstanceShare": incremental_last_modified, + "ConvMessageSendRequest": default, + "AppointmentInvitationFeed": default, + "EngagementChannelWorkType": default, + "PromotionLineItemRule": default, + "ShippingCarrier": default, + "MLModelMetric": default, + "PromotionTierHistory": incremental_created_date, + "IncidentRelatedItemHistory": incremental_created_date, + "ShippingRateGroup": default, + "ExternalDataSrcDescriptor": default, + "WaitlistWorkType": default, + "PrivacyJobSessionShare": incremental_last_modified, + "ReplyEmailSettings": default, + "ProblemRelatedItemHistory": incremental_created_date, + "AppointmentInvitationHistory": incremental_created_date, + "FlowRecordVersionOccurrence": default, + "ChangeRequest": default, + "MsgChannelUsageExternalOrg": default, + "ManagedContentVariant": default, + "FlowOrchestrationStepInstance": default, + "SellerShare": incremental_last_modified, + "PrivacyJobSession": default, + "WaitlistServiceResource": default, + "FlowOrchestrationStepInstanceShare": incremental_last_modified, + "FlowRecordVersion": default, + "ScorecardMetric": default, + "ChangeRequestRelatedItemFeed": default, + "UserDefinedLabelAssignmentShare": incremental_last_modified, + "ProblemRelatedItem": default, + "GuestUserAnomalyEventStoreFeed": default, + "EngagementChannelWorkTypeFeed": default, + "SkillType": default, + "InventoryReservationShare": incremental_last_modified, + "Problem": default, + "AppointmentInvitationShare": incremental_last_modified, + "ChatRetirementRdyMetrics": default, + "BusinessBrandShare": incremental_last_modified, + "SellerHistory": incremental_created_date, + "AppointmentCategoryFeed": default, + "ProblemIncidentFeed": default, + "ShippingConfigurationSet": default, + "MLModel": default, + "WaitlistWorkTypeHistory": incremental_created_date, + "ShippingCarrierFeed": default, + "PromotionSegmentBuyerGroupFeed": default, + "PrivacySessionRecordFailure": default, + "ContractLineItemFeed": default, + "ContextParamMap": default, + "FlowOrchestrationInstance": default, + "ActivityFieldHistory": default_full, + "EngagementChannelWorkTypeHistory": incremental_created_date, + "WaitlistParticipantHistory": incremental_created_date, + "WebCartAdjustmentBasis": default, + "ScorecardAssociation": default, + "ShippingConfigurationSetShare": incremental_last_modified, + "PromotionQualifierFeed": default, + "UserPrioritizedRecordShare": incremental_last_modified, + "CustomerShare": incremental_last_modified, + "WaitlistParticipant": default, + "BusinessBrand": default, + "PromotionLineItemRuleShare": incremental_last_modified, + "PrivacyObjectSession": default, + "FlowRecord": default, + "ShippingCarrierMethodHistory": incremental_created_date, + "PrivacyRTBFRequestHistory": incremental_created_date, + "TableauHostMapping": default, + "AppointmentInvitation": default, + "PipelineInspectionListView": default, + "PromotionTier": default, + "ServiceAppointmentAttendeeHistory": incremental_created_date, + "ProcessFlowMigration": default, + "ServiceAppointmentAttendeeShare": incremental_last_modified, + "Scorecard": default, + "ManagedContent": default, + "ChangeRequestRelatedItem": default, + "ShiftEngagementChannelFeed": default, + "EventRelayFeedback": default, + "CaseRelatedIssueFeed": default, + "ChangeRequestRelatedIssueHistory": incremental_created_date, + "Incident": default, + "EngmtChannelTypeSettings": default, + "PrivacyRTBFRequestShare": incremental_last_modified, + "IncidentRelatedItemFeed": default, + "ProblemHistory": incremental_created_date, + "ChangeRequestRelatedIssueFeed": default, + "CouponFeed": default, + "WaitlistWorkTypeFeed": default, + "GuestUserAnomalyEventStore": default, + "AppointmentCategoryHistory": incremental_created_date, + "OauthTokenExchHandlerApp": default, + "FlowOrchestrationWorkItem": default, + "CouponCodeRedemption": default, + "PromotionTierFeed": default, + "ExpressionSetView": incremental_last_modified, + "OpportunityRelatedDeleteLog": default, + "ShippingCarrierHistory": incremental_created_date, + "ShippingCarrierMethodFeed": default, + "ChangeRequestRelatedIssue": default, + "WaitlistShare": incremental_last_modified, + "Customer": default, + "EventRelayConfig": default, + "ShippingCarrierMethodShare": incremental_last_modified, + "PrivacyPolicyDefinition": default, + "ProblemFeed": default, + "FlowRecordElement": default, + "ProblemRelatedItemFeed": default, + "AssignedResourceHistory": incremental_created_date, + "UserPrioritizedRecord": default, + "CaseRelatedIssueHistory": incremental_created_date, + "FlowOrchestrationInstanceShare": incremental_last_modified, + "FlowOrchestrationLog": default, + "OauthTokenExchangeHandler": default, + "BrowserPolicyViolation": default, + "WaitlistServiceResourceHistory": incremental_created_date, + "WaitlistHistory": incremental_created_date, + "WaitlistServiceResourceFeed": default, + "ShippingRateArea": default, + "Seller": default, + "ServiceAppointmentAttendee": default, + "PrivacyRTBFRequestFeed": default, + "WaitlistParticipantFeed": default, + "ChangeRequestShare": incremental_last_modified, + "ChangeRequestHistory": incremental_created_date, + "CaseRelatedIssue": default, + "PendingOrderSummary": default_full, + "Waitlist": default, + "CaseHistory2": default, + "AppointmentInvitee": default, + "Participant": default, + "UserDefinedLabelAssignment": default, + "ManagedContentSpace": default, + "ProblemIncidentHistory": incremental_created_date, + "PromotionSegmentFeed": default, + "ManagedContentChannel": default, + "ShippingCarrierMethod": default, + "PrivacySessionRecordFailureShare": incremental_last_modified, + "InventoryReservation": default, + "PrivacyRTBFRequest": default, + "UserDefinedLabelShare": incremental_last_modified, + "FileEventStore": default_full, + "WebStoreInventorySource": default, + "FlowTestResult": default, + "IncidentRelatedItem": default, + "PromotionSegmentSalesStoreFeed": default, + "ScorecardShare": incremental_last_modified, + "WaitlistFeed": default, + "PrivacyObjectSessionShare": incremental_last_modified, + "ShiftWorkTopicHistory": incremental_created_date, + "CouponCodeRedemptionShare": incremental_last_modified, + "PrivacyPolicy": default, + "ShiftEngagementChannelHistory": incremental_created_date, + "IncidentHistory": incremental_created_date, + "OrgEmailAddressSecurity": default, + "ShippingCarrierShare": incremental_last_modified, + "UserDefinedLabel": default, + "MessagingChannelUsage": default, + "IncidentFeed": default, + "TableauHostMappingShare": incremental_last_modified, + "ShiftWorkTopic": default, + "ChangeRequestRelatedItemHistory": incremental_created_date, + "MLRecommendationDefinition": default, + "PaymentFeed": default, + "MLModelFactorComponent": default, + "ChangeRequestFeed": default, + "MLModelFactor": default, + "InventoryItemReservation": default, + "PromotionTargetFeed": default, + "ShiftEngagementChannel": default, + "ProblemIncident": default, + "ShiftWorkTopicFeed": default, + "FlowRecordShare": incremental_last_modified, + "IncidentShare": incremental_last_modified, + "AppointmentCategory": default, + "StandardShippingRate": default, + "MlFeatureValueMetric": default, + "FlowOrchestrationWorkItemShare": incremental_last_modified, + "DataWeaveResource": default, + "ExternalEncryptionRootKey": default_full, + "ServiceAppointmentAttendeeFeed": default, + "FlowOrchestrationStageInstance": default, + "ExtlClntAppOauthPlcyAttr": default, + "LocationShippingCarrierMethodFeed": default, + "ExtlClntAppOauthPlcyAttr": default, + "ExtlClntAppOauthIpRange": default, + "ExtlClntAppOauthConsumer": default, + "ExtlClntAppPlcyCnfg": default, + "DeliveryEstimationSetupShare": incremental_last_modified, + "ConvIntelligenceSignalSubRule": default, + "ConvIntelligenceSignalRule": default, + "ExtlClntAppSampleSettings": default, + "ExtlClntAppOauthPlcyCnfg": default, + "ExtlClntAppOauthSettings": default, + "ExtlClntAppOauthSetCustmScp": default, + "DeliveryEstimationSetup": default, + "LocationShippingCarrierMethodShare": incremental_last_modified, + "ExternalClientApplication": default, + "LocationShippingCarrierMethodHistory": incremental_created_date, + "ExtlClntAppSamplePlcyCnfg": default, + "DeliveryEstimationSetupFeed": default, + "ExtlClntAppOauthPlcyCustmScp": default, + "ExtlClntAppOauthSetAttr": default, + "LocationShippingCarrierMethod": default, + "DeliveryEstimationSetupHistory": incremental_created_date, # Added on 10/24/25 - 'AddressHistory': incremental_created_date, - 'OrgMetric': default, - 'OrgMetricScanResult': default, - 'OrgMetricScanSummary': default + "AddressHistory": incremental_created_date, + "OrgMetric": default, + "OrgMetricScanResult": default, + "OrgMetricScanSummary": default, } def rest_only_streams(self): """A group of streams that is only discovered when the REST API is in use.""" return { - 'CaseStatus', - 'DeclinedEventRelation', - 'RecentlyViewed', - 'SolutionStatus', - 'TaskStatus', - 'OrderStatus', - 'WorkStepStatus', - 'FieldSecurityClassification', - 'AcceptedEventRelation', - 'ContractStatus', - 'PartnerRole', - 'WorkOrderStatus', - 'ShiftStatus', - 'WorkOrderLineItemStatus', - 'ServiceAppointmentStatus', - 'TaskPriority', - 'UndecidedEventRelation', + "CaseStatus", + "DeclinedEventRelation", + "RecentlyViewed", + "SolutionStatus", + "TaskStatus", + "OrderStatus", + "WorkStepStatus", + "FieldSecurityClassification", + "AcceptedEventRelation", + "ContractStatus", + "PartnerRole", + "WorkOrderStatus", + "ShiftStatus", + "WorkOrderLineItemStatus", + "ServiceAppointmentStatus", + "TaskPriority", + "UndecidedEventRelation", } def expected_streams(self): """A set of expected stream names""" streams = set(self.expected_metadata().keys()) - if self.salesforce_api == 'BULK': + if self.salesforce_api == "BULK": return streams.difference(self.rest_only_streams()) return streams @@ -1136,56 +1144,70 @@ def child_streams(self): Return a set of streams that are child streams based on having foreign key metadata """ - return {stream for stream, metadata in self.expected_metadata().items() - if metadata.get(self.FOREIGN_KEYS)} + return { + stream + for stream, metadata in self.expected_metadata().items() + if metadata.get(self.FOREIGN_KEYS) + } def expected_primary_keys(self): """ return a dictionary with key of table name and value as a set of primary key fields """ - return {table: properties.get(self.PRIMARY_KEYS, set()) - for table, properties - in self.expected_metadata().items()} + return { + table: properties.get(self.PRIMARY_KEYS, set()) + for table, properties in self.expected_metadata().items() + } def expected_replication_keys(self): """ return a dictionary with key of table name and value as a set of replication key fields """ - return {table: properties.get(self.REPLICATION_KEYS, set()) - for table, properties - in self.expected_metadata().items()} + return { + table: properties.get(self.REPLICATION_KEYS, set()) + for table, properties in self.expected_metadata().items() + } def expected_foreign_keys(self): """ return a dictionary with key of table name and value as a set of foreign key fields """ - return {table: properties.get(self.FOREIGN_KEYS, set()) - for table, properties - in self.expected_metadata().items()} - + return { + table: properties.get(self.FOREIGN_KEYS, set()) + for table, properties in self.expected_metadata().items() + } def expected_automatic_fields(self): auto_fields = {} for k, v in self.expected_metadata().items(): - auto_fields[k] = v.get(self.PRIMARY_KEYS, set()) | v.get(self.REPLICATION_KEYS, set()) \ + auto_fields[k] = ( + v.get(self.PRIMARY_KEYS, set()) + | v.get(self.REPLICATION_KEYS, set()) | v.get(self.FOREIGN_KEYS, set()) + ) return auto_fields def expected_replication_method(self): """return a dictionary with key of table name nd value of replication method""" - return {table: properties.get(self.REPLICATION_METHOD, None) - for table, properties - in self.expected_metadata().items()} + return { + table: properties.get(self.REPLICATION_METHOD, None) + for table, properties in self.expected_metadata().items() + } def setUp(self): """Verify that you have set the prerequisites to run the tap (creds, etc.)""" - missing_envs = [x for x in ['TAP_SALESFORCE_CLIENT_ID', - 'TAP_SALESFORCE_CLIENT_SECRET', - 'TAP_SALESFORCE_REFRESH_TOKEN'] - if os.getenv(x) is None] + missing_envs = [ + x + for x in [ + "TAP_SALESFORCE_CLIENT_ID", + "TAP_SALESFORCE_CLIENT_SECRET", + "TAP_SALESFORCE_REFRESH_TOKEN", + ] + if os.getenv(x) is None + ] if missing_envs: raise Exception("set environment variables") @@ -1209,7 +1231,11 @@ def run_and_verify_check_mode(self, conn_id): menagerie.verify_check_exit_status(self, exit_status, check_job_name) found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id)) + self.assertGreater( + len(found_catalogs), + 0, + msg="unable to locate schemas for connection {}".format(conn_id), + ) return found_catalogs @@ -1228,19 +1254,20 @@ def run_and_verify_sync(self, conn_id): # Verify actual rows were synced sync_record_count = runner.examine_target_output_file( - self, conn_id, self.expected_streams(), self.expected_primary_keys()) + self, conn_id, self.expected_streams(), self.expected_primary_keys() + ) self.assertGreater( - sum(sync_record_count.values()), 0, - msg="failed to replicate any data: {}".format(sync_record_count) + sum(sync_record_count.values()), + 0, + msg="failed to replicate any data: {}".format(sync_record_count), ) LOGGER.info("total replicated row count: %s", sum(sync_record_count.values())) return sync_record_count - def perform_and_verify_table_and_field_selection(self, - conn_id, - test_catalogs, - select_all_fields=True): + def perform_and_verify_table_and_field_selection( + self, conn_id, test_catalogs, select_all_fields=True + ): """ Perform table and field selection based off of the streams to select set and field selection parameters. @@ -1257,79 +1284,116 @@ def perform_and_verify_table_and_field_selection(self, catalogs = menagerie.get_catalogs(conn_id) # Ensure our selection affects the catalog - expected_selected = [tc.get('tap_stream_id') for tc in test_catalogs] + expected_selected = [tc.get("tap_stream_id") for tc in test_catalogs] for cat in catalogs: - catalog_entry = menagerie.get_annotated_schema(conn_id, cat['stream_id']) + catalog_entry = menagerie.get_annotated_schema(conn_id, cat["stream_id"]) # Verify all testable streams are selected - selected = catalog_entry.get('annotated-schema').get('selected') - LOGGER.info("Validating selection on %s: %s", cat['stream_name'], selected) - if cat['stream_name'] not in expected_selected: + selected = catalog_entry.get("annotated-schema").get("selected") + LOGGER.info("Validating selection on %s: %s", cat["stream_name"], selected) + if cat["stream_name"] not in expected_selected: self.assertFalse(selected, msg="Stream selected, but not testable.") - continue # Skip remaining assertions if we aren't selecting this stream + continue # Skip remaining assertions if we aren't selecting this stream self.assertTrue(selected, msg="Stream not selected.") if select_all_fields: # Verify all fields within each selected stream are selected - for field, field_props in catalog_entry.get('annotated-schema').get('properties').items(): - field_selected = field_props.get('selected') - LOGGER.info("\tValidating selection on %s.%s: %s", - cat['stream_name'], field, field_selected) + for field, field_props in ( + catalog_entry.get("annotated-schema").get("properties").items() + ): + field_selected = field_props.get("selected") + LOGGER.info( + "\tValidating selection on %s.%s: %s", + cat["stream_name"], + field, + field_selected, + ) self.assertTrue(field_selected, msg="Field not selected.") else: # Verify only automatic fields are selected - expected_automatic_fields = self.expected_automatic_fields().get(cat['tap_stream_id']) - selected_fields = self.get_selected_fields_from_metadata(catalog_entry['metadata']) + expected_automatic_fields = self.expected_automatic_fields().get( + cat["tap_stream_id"] + ) + selected_fields = self.get_selected_fields_from_metadata( + catalog_entry["metadata"] + ) self.assertEqual(expected_automatic_fields, selected_fields) @staticmethod def get_selected_fields_from_metadata(metadata): selected_fields = set() for field in metadata: - is_field_metadata = len(field['breadcrumb']) > 1 - if field['metadata'].get('inclusion') is None and is_field_metadata: # BUG_SRCE-4313 remove when addressed - LOGGER.info("Error %s has no inclusion key in metadata", field) # BUG_SRCE-4313 remove when addressed + is_field_metadata = len(field["breadcrumb"]) > 1 + if ( + field["metadata"].get("inclusion") is None and is_field_metadata + ): # BUG_SRCE-4313 remove when addressed + LOGGER.info( + "Error %s has no inclusion key in metadata", field + ) # BUG_SRCE-4313 remove when addressed continue # BUG_SRCE-4313 remove when addressed inclusion_automatic_or_selected = ( - field['metadata']['selected'] is True or \ - field['metadata']['inclusion'] == 'automatic' + field["metadata"]["selected"] is True + or field["metadata"]["inclusion"] == "automatic" ) if is_field_metadata and inclusion_automatic_or_selected: - selected_fields.add(field['breadcrumb'][1]) + selected_fields.add(field["breadcrumb"][1]) return selected_fields - @staticmethod - def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = True): + def select_all_streams_and_fields( + conn_id, catalogs, select_all_fields: bool = True + ): """Select all streams and all fields within streams""" for catalog in catalogs: - schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) + schema = menagerie.get_annotated_schema(conn_id, catalog["stream_id"]) non_selected_properties = [] if not select_all_fields: # get a list of all properties so that none are selected - non_selected_properties = schema.get('annotated-schema', {}).get( - 'properties', {}).keys() + non_selected_properties = ( + schema.get("annotated-schema", {}).get("properties", {}).keys() + ) connections.select_catalog_and_fields_via_metadata( - conn_id, catalog, schema, [], non_selected_properties) + conn_id, catalog, schema, [], non_selected_properties + ) def set_replication_methods(self, conn_id, catalogs, replication_methods): - replication_keys = self.expected_replication_keys() for catalog in catalogs: - - replication_method = replication_methods.get(catalog['stream_name']) + replication_method = replication_methods.get(catalog["stream_name"]) if replication_method == self.INCREMENTAL: - replication_key = list(replication_keys.get(catalog['stream_name']))[0] - replication_md = [{ "breadcrumb": [], "metadata": {'replication-key': replication_key, "replication-method" : replication_method, "selected" : True}}] + replication_key = list(replication_keys.get(catalog["stream_name"]))[0] + replication_md = [ + { + "breadcrumb": [], + "metadata": { + "replication-key": replication_key, + "replication-method": replication_method, + "selected": True, + }, + } + ] else: - replication_md = [{ "breadcrumb": [], "metadata": {'replication-key': None, "replication-method" : "FULL_TABLE", "selected" : True}}] + replication_md = [ + { + "breadcrumb": [], + "metadata": { + "replication-key": None, + "replication-method": "FULL_TABLE", + "selected": True, + }, + } + ] connections.set_non_discoverable_metadata( - conn_id, catalog, menagerie.get_annotated_schema(conn_id, catalog['stream_id']), replication_md) + conn_id, + catalog, + menagerie.get_annotated_schema(conn_id, catalog["stream_id"]), + replication_md, + ) @staticmethod def parse_date(date_value): @@ -1345,14 +1409,22 @@ def parse_date(date_value): return date_stripped except ValueError: try: - date_stripped = dt.strptime(date_value, "%Y-%m-%dT%H:%M:%S.%f+00:00") + date_stripped = dt.strptime( + date_value, "%Y-%m-%dT%H:%M:%S.%f+00:00" + ) return date_stripped except ValueError: try: - date_stripped = dt.strptime(date_value, "%Y-%m-%dT%H:%M:%S+00:00") + date_stripped = dt.strptime( + date_value, "%Y-%m-%dT%H:%M:%S+00:00" + ) return date_stripped except ValueError as e_final: - raise NotImplementedError("We are not accounting for dates of this format: {}".format(date_value)) from e_final + raise NotImplementedError( + "We are not accounting for dates of this format: {}".format( + date_value + ) + ) from e_final def timedelta_formatted(self, dtime, days=0): try: @@ -1369,7 +1441,11 @@ def timedelta_formatted(self, dtime, days=0): return dt.strftime(return_date, self.BOOKMARK_COMPARISON_FORMAT) except ValueError: - return Exception("Datetime object is not of the format: {}".format(self.START_DATE_FORMAT)) + return Exception( + "Datetime object is not of the format: {}".format( + self.START_DATE_FORMAT + ) + ) ########################################################################## ### Tap Specific Methods @@ -1379,63 +1455,63 @@ def timedelta_formatted(self, dtime, days=0): def get_unsupported_by_rest_api(): """The streams listed here are not supported by the REST API""" unsupported_streams = { - 'Announcement', - 'CollaborationGroupRecord', - 'ContentDocumentLink', - 'ContentFolderMember', - 'DataStatistics', - 'EntityParticle', - 'FieldDefinition', - 'FlexQueueItem', - 'IdeaComment', - 'OwnerChangeOptionInfo', - 'PicklistValueInfo', - 'PlatformAction', - 'RelationshipDomain', - 'RelationshipInfo', - 'SearchLayout', - 'SiteDetail', - 'UserEntityAccess', - 'UserFieldAccess', - 'Vote', - 'RecordActionHistory', - 'FlowVersionView', - 'FlowVariableView', - 'AppTabMember', - 'ColorDefinition', - 'IconDefinition', + "Announcement", + "CollaborationGroupRecord", + "ContentDocumentLink", + "ContentFolderMember", + "DataStatistics", + "EntityParticle", + "FieldDefinition", + "FlexQueueItem", + "IdeaComment", + "OwnerChangeOptionInfo", + "PicklistValueInfo", + "PlatformAction", + "RelationshipDomain", + "RelationshipInfo", + "SearchLayout", + "SiteDetail", + "UserEntityAccess", + "UserFieldAccess", + "Vote", + "RecordActionHistory", + "FlowVersionView", + "FlowVariableView", + "AppTabMember", + "ColorDefinition", + "IconDefinition", } return unsupported_streams def get_unsupported_by_bulk_api(self): unsupported_streams_rest = self.get_unsupported_by_rest_api() - unsupported_streams_bulk_only= { - 'AcceptedEventRelation', - 'AssetTokenEvent', - 'AttachedContentNote', - 'CaseStatus', - 'ContentFolderItem', - 'ContractStatus', - 'DeclinedEventRelation', - 'EventWhoRelation', - 'PartnerRole', - 'QuoteTemplateRichTextData', - 'RecentlyViewed', - 'SolutionStatus', - 'TaskPriority', - 'TaskWhoRelation', - 'TaskStatus', - 'UndecidedEventRelation', - 'OrderStatus', - 'WorkOrderStatus', - 'WorkOrderLineItemStatus', - 'ServiceAppointmentStatus', - 'ServiceAppointmentStatus', - 'FieldSecurityClassification', + unsupported_streams_bulk_only = { + "AcceptedEventRelation", + "AssetTokenEvent", + "AttachedContentNote", + "CaseStatus", + "ContentFolderItem", + "ContractStatus", + "DeclinedEventRelation", + "EventWhoRelation", + "PartnerRole", + "QuoteTemplateRichTextData", + "RecentlyViewed", + "SolutionStatus", + "TaskPriority", + "TaskWhoRelation", + "TaskStatus", + "UndecidedEventRelation", + "OrderStatus", + "WorkOrderStatus", + "WorkOrderLineItemStatus", + "ServiceAppointmentStatus", + "ServiceAppointmentStatus", + "FieldSecurityClassification", # BUG_TODO | the following streams are undocumented - 'WorkStepStatus', - 'ShiftStatus', + "WorkStepStatus", + "ShiftStatus", } return unsupported_streams_bulk_only | unsupported_streams_rest diff --git a/tests/sfbase.py b/tests/sfbase.py index f52244d5..9ea941d0 100644 --- a/tests/sfbase.py +++ b/tests/sfbase.py @@ -3,12 +3,11 @@ Setup expectations for test sub classes Run discovery for as a prerequisite for most tests """ -import unittest -import os import math -from datetime import timedelta +import os from datetime import datetime as dt -from tap_tester import connections, menagerie, runner, LOGGER + +from tap_tester import LOGGER, connections, menagerie, runner from tap_tester.base_suite_tests.base_case import BaseCase diff --git a/tests/test_custom_objects_bulk.py b/tests/test_custom_objects_bulk.py index c19ce006..640d8203 100644 --- a/tests/test_custom_objects_bulk.py +++ b/tests/test_custom_objects_bulk.py @@ -4,7 +4,7 @@ class SalesforceCustomObjectsBulk(SalesforceCustomObjects): """Test that all fields can be replicated for a stream that is a custom object (BULK API)""" - salesforce_api = 'BULK' + salesforce_api = "BULK" @staticmethod def name(): diff --git a/tests/test_custom_objects_rest.py b/tests/test_custom_objects_rest.py index 77a3e582..9e79e2ea 100644 --- a/tests/test_custom_objects_rest.py +++ b/tests/test_custom_objects_rest.py @@ -1,11 +1,10 @@ """ Test that all fields can be replicated for a stream that is a custom object (REST API) """ -from datetime import datetime, timedelta -from tap_tester import connections, menagerie, runner from sfbase import SFBaseTest +from tap_tester import connections, menagerie, runner class SalesforceCustomObjects(SFBaseTest): @@ -13,12 +12,12 @@ class SalesforceCustomObjects(SFBaseTest): # hard code start date to replicate single custom record until CRUD is implemented # 'SystemModstamp': '2023-08-04T20:13:50.000000Z', # record info - start_date = '2023-08-04T00:00:00Z' + start_date = "2023-08-04T00:00:00Z" @staticmethod def expected_sync_streams(): return { - 'TapTester__c', # 1 created record present, spike on CRUD? + "TapTester__c", # 1 created record present, spike on CRUD? # 'TapTester__Share', # TODO no data, fix or omit? } @@ -39,36 +38,39 @@ def custom_objects_test(self): found_catalogs = self.run_and_verify_check_mode(conn_id) # table and field selection - test_catalogs = [catalog for catalog in found_catalogs - if catalog.get('stream_name') in expected_streams] + test_catalogs = [ + catalog + for catalog in found_catalogs + if catalog.get("stream_name") in expected_streams + ] self.perform_and_verify_table_and_field_selection(conn_id, test_catalogs) # set additional metadata to define replication key for the custom stream catalog = test_catalogs[0] - schema_and_metadata = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) + schema_and_metadata = menagerie.get_annotated_schema( + conn_id, catalog["stream_id"] + ) non_selected_fields = [] - additional_md = [{"breadcrumb": [], - "metadata": { - "replication-key": "SystemModstamp" - }}] + additional_md = [ + {"breadcrumb": [], "metadata": {"replication-key": "SystemModstamp"}} + ] connections.select_catalog_and_fields_via_metadata( - conn_id, - catalog, - schema_and_metadata, - additional_md, - non_selected_fields) + conn_id, catalog, schema_and_metadata, additional_md, non_selected_fields + ) # grab metadata after performing table-and-field selection to set expectations, # used for asserting all fields are replicated stream_to_all_catalog_fields = dict() for catalog in test_catalogs: - stream_id, stream_name = catalog['stream_id'], catalog['stream_name'] + stream_id, stream_name = catalog["stream_id"], catalog["stream_name"] catalog_entry = menagerie.get_annotated_schema(conn_id, stream_id) - fields_from_field_level_md = [md_entry['breadcrumb'][1] - for md_entry in catalog_entry['metadata'] - if md_entry['breadcrumb'] != []] + fields_from_field_level_md = [ + md_entry["breadcrumb"][1] + for md_entry in catalog_entry["metadata"] + if md_entry["breadcrumb"] != [] + ] stream_to_all_catalog_fields[stream_name] = set(fields_from_field_level_md) # run initial sync @@ -77,19 +79,23 @@ def custom_objects_test(self): for stream in expected_streams: with self.subTest(stream=stream): - # expected values expected_keys = stream_to_all_catalog_fields.get(stream) # collect actual values data = synced_records.get(stream) - record_messages_keys = [set(row['data'].keys()) for row in data['messages'] - if row['action'] == 'upsert'] + record_messages_keys = [ + set(row["data"].keys()) + for row in data["messages"] + if row["action"] == "upsert" + ] # Verify that you get some records for each stream self.assertGreater( - record_count_by_stream.get(stream, -1), 0, - msg="The number of records is not over the stream max limit") + record_count_by_stream.get(stream, -1), + 0, + msg="The number of records is not over the stream max limit", + ) # Verify that all selected / expected fields are sent to the target for actual_keys in record_messages_keys: @@ -99,7 +105,7 @@ def custom_objects_test(self): class SalesforceCustomObjectsRest(SalesforceCustomObjects): """Test that all fields can be replicated for a stream that is a custom object (REST API)""" - salesforce_api = 'REST' + salesforce_api = "REST" @staticmethod def name(): diff --git a/tests/test_salesforce_activate_version_messages.py b/tests/test_salesforce_activate_version_messages.py index afe9f9d8..3619aee6 100644 --- a/tests/test_salesforce_activate_version_messages.py +++ b/tests/test_salesforce_activate_version_messages.py @@ -1,7 +1,5 @@ -import unittest -from tap_tester import runner, menagerie, connections - from base import SalesforceBaseTest +from tap_tester import connections, menagerie, runner class SalesforceActivateVersionMessages(SalesforceBaseTest): @@ -12,11 +10,11 @@ def name(): @staticmethod def expected_sync_streams(): return { - 'Account', - 'Contact', - 'Lead', - 'Opportunity', - 'User', + "Account", + "Contact", + "Lead", + "Opportunity", + "User", } def test_run(self): @@ -33,32 +31,43 @@ def test_run(self): 4. start a new full table - should emit activate version message at end with new version """ - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" conn_id = connections.ensure_connection(self) - #run in check mode + # run in check mode found_catalogs = self.run_and_verify_check_mode(conn_id) # select streams expected_streams = self.expected_sync_streams() - catalog_entries = [catalog for catalog in found_catalogs - if catalog.get('tap_stream_id') in expected_streams] - self.perform_and_verify_table_and_field_selection(conn_id, catalog_entries, select_all_fields=True) + catalog_entries = [ + catalog + for catalog in found_catalogs + if catalog.get("tap_stream_id") in expected_streams + ] + self.perform_and_verify_table_and_field_selection( + conn_id, catalog_entries, select_all_fields=True + ) # run initial sync (initial incremental) - streams_replication_methods = {stream: self.INCREMENTAL - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) - menagerie.set_state(conn_id, {}) # clear state + streams_replication_methods = { + stream: self.INCREMENTAL for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) + menagerie.set_state(conn_id, {}) # clear state _ = self.run_and_verify_sync(conn_id) initial_incremental_records = runner.get_records_from_target_output() # run a second sync (initial full table) - streams_replication_methods = {stream: self.FULL_TABLE - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) + streams_replication_methods = { + stream: self.FULL_TABLE for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) _ = self.run_and_verify_sync(conn_id) initial_full_table_records = runner.get_records_from_target_output() @@ -66,43 +75,63 @@ def test_run(self): _ = self.run_and_verify_sync(conn_id) final_full_table_records = runner.get_records_from_target_output() - # Assertions made by stream for stream in expected_streams: with self.subTest(stream=stream): - # Sync 1 (inital incremental) - initial_incremental_message = initial_incremental_records[stream]['messages'][0] - incremental_version = initial_incremental_records[stream]['table_version'] + initial_incremental_message = initial_incremental_records[stream][ + "messages" + ][0] + incremental_version = initial_incremental_records[stream][ + "table_version" + ] - self.assertEqual(initial_incremental_message['action'], - 'activate_version', - msg="Expected `activate_version` message to be sent prior to records for incremental sync") + self.assertEqual( + initial_incremental_message["action"], + "activate_version", + msg="Expected `activate_version` message to be sent prior to records for incremental sync", + ) # Sync 2 (inital full table) - initial_full_table_message = initial_full_table_records[stream]['messages'][0] - initial_full_table_version = initial_full_table_records[stream]['table_version'] - - self.assertEqual(initial_full_table_message['action'], - 'activate_version', - msg="Expected `activate_version` message to be sent prior to records for initial full table sync") - - self.assertNotEqual(initial_full_table_version, - incremental_version, - msg="Expected version for stream Account to be change after switching to full table") + initial_full_table_message = initial_full_table_records[stream][ + "messages" + ][0] + initial_full_table_version = initial_full_table_records[stream][ + "table_version" + ] + + self.assertEqual( + initial_full_table_message["action"], + "activate_version", + msg="Expected `activate_version` message to be sent prior to records for initial full table sync", + ) + + self.assertNotEqual( + initial_full_table_version, + incremental_version, + msg="Expected version for stream Account to be change after switching to full table", + ) # Sync 3 (final full table) # final_full_table_initial_message = final_full_table_records[stream]['messages'][0] - final_full_table_final_message = final_full_table_records[stream]['messages'][-1] - final_full_table_version = final_full_table_records[stream]['table_version'] - - self.assertEqual(final_full_table_final_message['action'], - 'activate_version', - msg="Expected `activate_version` message to be sent after records for subsequent full table syncs") - - self.assertNotEqual(final_full_table_version, - initial_full_table_version, - msg="Expected version for stream Account to be change after switching to full table") + final_full_table_final_message = final_full_table_records[stream][ + "messages" + ][-1] + final_full_table_version = final_full_table_records[stream][ + "table_version" + ] + + self.assertEqual( + final_full_table_final_message["action"], + "activate_version", + msg="Expected `activate_version` message to be sent after records for subsequent full table syncs", + ) + + self.assertNotEqual( + final_full_table_version, + initial_full_table_version, + msg="Expected version for stream Account to be change after switching to full table", + ) diff --git a/tests/test_salesforce_all_fields_custom.py b/tests/test_salesforce_all_fields_custom.py index 1520ddbc..764def62 100644 --- a/tests/test_salesforce_all_fields_custom.py +++ b/tests/test_salesforce_all_fields_custom.py @@ -1,8 +1,9 @@ """ Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated """ -from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest from sfbase import SFBaseTest +from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest + class SFCustomFieldsTest(AllFieldsTest, SFBaseTest): @@ -25,7 +26,7 @@ def test_all_fields_for_streams_are_replicated(self): selected_streams = self.streams_to_test() actual_custom_field_streams = {key for key in self.selected_fields.keys() if self.selected_fields.get(key,set())} self.assertSetEqual( selected_streams, actual_custom_field_streams, - msg = f"More streams have custom fields actual_custom_field_streams.diff(selected_streams)") + msg = "More streams have custom fields actual_custom_field_streams.diff(selected_streams)") for stream in selected_streams: with self.subTest(stream=stream): automatic_fields = self.expected_automatic_fields(stream) diff --git a/tests/test_salesforce_all_fields_custom_rest.py b/tests/test_salesforce_all_fields_custom_rest.py index e4877dad..6105fafb 100644 --- a/tests/test_salesforce_all_fields_custom_rest.py +++ b/tests/test_salesforce_all_fields_custom_rest.py @@ -1,19 +1,20 @@ """ Test that with only custom fields selected for a stream automatic fields and custom fields are still replicated """ -from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest + from sfbase import SFBaseTest +from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest -class SFCustomFieldsTestRest(AllFieldsTest, SFBaseTest): - salesforce_api = 'REST' +class SFCustomFieldsTestRest(AllFieldsTest, SFBaseTest): + salesforce_api = "REST" @staticmethod def name(): return "tt_sf_all_fields_custom_rest" def streams_to_test(self): - return self.get_custom_fields_streams() + return self.get_custom_fields_streams() def streams_to_selected_fields(self): found_catalogs = AllFieldsTest.found_catalogs @@ -22,27 +23,46 @@ def streams_to_selected_fields(self): return custom_fields def test_all_fields_for_streams_are_replicated(self): - selected_streams = self.streams_to_test() - actual_custom_field_streams = {key for key in self.selected_fields.keys() if self.selected_fields.get(key,set())} - self.assertSetEqual( selected_streams, actual_custom_field_streams, - msg = f"More streams have custom fields actual_custom_field_streams.diff(selected_streams)") + actual_custom_field_streams = { + key + for key in self.selected_fields.keys() + if self.selected_fields.get(key, set()) + } + self.assertSetEqual( + selected_streams, + actual_custom_field_streams, + msg="More streams have custom fields actual_custom_field_streams.diff(selected_streams)", + ) for stream in selected_streams: with self.subTest(stream=stream): automatic_fields = self.expected_automatic_fields(stream) - expected_custom_fields = self.selected_fields.get(stream, set()).union(automatic_fields) + expected_custom_fields = self.selected_fields.get(stream, set()).union( + automatic_fields + ) replicated_custom_fields = self.actual_fields.get(stream, set()) - #Verify that custom and automatic fields are replicated - self.assertSetEqual(expected_custom_fields, replicated_custom_fields, - msg = f"verify all fields are replicated for stream {stream}") + # Verify that custom and automatic fields are replicated + self.assertSetEqual( + expected_custom_fields, + replicated_custom_fields, + msg=f"verify all fields are replicated for stream {stream}", + ) - #Verify at least one custom field is replicated + # Verify at least one custom field is replicated if len(expected_custom_fields) > len(automatic_fields): - self.assertGreater(len(replicated_custom_fields.difference(automatic_fields)),0, - msg = f"Replication didn't return any custom fields for stream {stream}") + self.assertGreater( + len(replicated_custom_fields.difference(automatic_fields)), + 0, + msg=f"Replication didn't return any custom fields for stream {stream}", + ) - #Verify that only custom fields are replicated besides automatic fields - _, num_non_custom = self.count_custom_non_custom_fields(replicated_custom_fields) - self.assertEqual(num_non_custom, len(automatic_fields), - msg = f"Replicated some fields that are not custom fields for stream {stream}") + # Verify that only custom fields are replicated besides automatic fields + _, num_non_custom = self.count_custom_non_custom_fields( + replicated_custom_fields + ) + self.assertEqual( + num_non_custom, + len(automatic_fields), + msg=f"Replicated some fields that are not custom fields for stream {stream}", + ) diff --git a/tests/test_salesforce_all_fields_non_custom.py b/tests/test_salesforce_all_fields_non_custom.py index 159e74bb..70baf966 100644 --- a/tests/test_salesforce_all_fields_non_custom.py +++ b/tests/test_salesforce_all_fields_non_custom.py @@ -1,14 +1,14 @@ """ Test that with only non-custom fields selected for a stream automatic fields and non custom fields are still replicated """ + +from sfbase import SFBaseTest from tap_tester import menagerie, runner from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest -from sfbase import SFBaseTest class SFNonCustomFieldsTest(AllFieldsTest, SFBaseTest): - - salesforce_api = 'BULK' + salesforce_api = "BULK" @staticmethod def name(): @@ -16,12 +16,12 @@ def name(): def streams_to_test(self): return { - 'Account', - 'ActiveProfileMetric', - 'Calendar', - 'ContentWorkspacePermission', - 'CampaignMemberStatus', - 'Community', + "Account", + "ActiveProfileMetric", + "Calendar", + "ContentWorkspacePermission", + "CampaignMemberStatus", + "Community", } def run_and_verify_check_mode(self, conn_id): @@ -40,14 +40,17 @@ def run_and_verify_check_mode(self, conn_id): # Verify the catalog is not empty found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0, - logging="A catalog was produced by discovery.") + self.assertGreater( + len(found_catalogs), 0, logging="A catalog was produced by discovery." + ) # TODO do we want this? # Verify the expected streams are present in the catalog - found_stream_names = {catalog['stream_name'] for catalog in found_catalogs} - self.assertTrue(self.expected_stream_names().issubset(found_stream_names), - logging="Expected streams are present in catalog.") + found_stream_names = {catalog["stream_name"] for catalog in found_catalogs} + self.assertTrue( + self.expected_stream_names().issubset(found_stream_names), + logging="Expected streams are present in catalog.", + ) return found_catalogs @@ -60,22 +63,37 @@ def streams_to_selected_fields(self): def test_non_custom_fields(self): for stream in self.streams_to_test(): with self.subTest(stream=stream): - expected_non_custom_fields = self.selected_fields.get(stream,set()) + expected_non_custom_fields = self.selected_fields.get(stream, set()) replicated_non_custom_fields = self.actual_fields.get(stream, set()) - #Verify at least one non-custom field is replicated - self.assertGreater(len(replicated_non_custom_fields),0, - msg = f"Replication didn't return any non-custom fields for stream {stream}") - - #verify that all the non_custom fields are replicated - self.assertEqual(replicated_non_custom_fields, expected_non_custom_fields, - msg = f"All non_custom fields are not no replicated for stream {stream}") - - #verify that automatic fields are also replicated along with non_custom_fields - self.assertTrue(self.expected_automatic_fields(stream).issubset(replicated_non_custom_fields), - msg = f"Automatic fields are not replicated for stream {stream}") - - #Verify custom fields are not replicated by checking the field name - num_custom, _ = self.count_custom_non_custom_fields(replicated_non_custom_fields) - self.assertEqual(num_custom, 0, - msg = f"replicated some fields that are custom for stream {stream}") + # Verify at least one non-custom field is replicated + self.assertGreater( + len(replicated_non_custom_fields), + 0, + msg=f"Replication didn't return any non-custom fields for stream {stream}", + ) + + # verify that all the non_custom fields are replicated + self.assertEqual( + replicated_non_custom_fields, + expected_non_custom_fields, + msg=f"All non_custom fields are not no replicated for stream {stream}", + ) + + # verify that automatic fields are also replicated along with non_custom_fields + self.assertTrue( + self.expected_automatic_fields(stream).issubset( + replicated_non_custom_fields + ), + msg=f"Automatic fields are not replicated for stream {stream}", + ) + + # Verify custom fields are not replicated by checking the field name + num_custom, _ = self.count_custom_non_custom_fields( + replicated_non_custom_fields + ) + self.assertEqual( + num_custom, + 0, + msg=f"replicated some fields that are custom for stream {stream}", + ) diff --git a/tests/test_salesforce_all_fields_non_custom_rest.py b/tests/test_salesforce_all_fields_non_custom_rest.py index 55d2c7ef..85897281 100644 --- a/tests/test_salesforce_all_fields_non_custom_rest.py +++ b/tests/test_salesforce_all_fields_non_custom_rest.py @@ -1,14 +1,14 @@ """ Test that with only non-custom fields selected for a stream automatic fields and non custom fields are still replicated """ -from tap_tester import menagerie, runner, LOGGER -from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest + from sfbase import SFBaseTest +from tap_tester import LOGGER, menagerie, runner +from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest class SFNonCustomFieldsTestRest(AllFieldsTest, SFBaseTest): - - salesforce_api = 'REST' + salesforce_api = "REST" @staticmethod def name(): @@ -16,12 +16,12 @@ def name(): def streams_to_test(self): return { - 'Case', - 'PricebookEntry', - 'Profile', - 'PermissionSet', - 'Product2', - 'PromptAction', + "Case", + "PricebookEntry", + "Profile", + "PermissionSet", + "Product2", + "PromptAction", } def run_and_verify_check_mode(self, conn_id): @@ -40,14 +40,17 @@ def run_and_verify_check_mode(self, conn_id): # Verify the catalog is not empty found_catalogs = menagerie.get_catalogs(conn_id) - self.assertGreater(len(found_catalogs), 0, - logging="A catalog was produced by discovery.") + self.assertGreater( + len(found_catalogs), 0, logging="A catalog was produced by discovery." + ) # TODO do we want this? # Verify the expected streams are present in the catalog - found_stream_names = {catalog['stream_name'] for catalog in found_catalogs} - self.assertTrue(self.expected_stream_names().issubset(found_stream_names), - logging="Expected streams are present in catalog.") + found_stream_names = {catalog["stream_name"] for catalog in found_catalogs} + self.assertTrue( + self.expected_stream_names().issubset(found_stream_names), + logging="Expected streams are present in catalog.", + ) return found_catalogs @@ -58,27 +61,46 @@ def streams_to_selected_fields(self): return non_custom_fields def test_non_custom_fields(self): - excluded_fields = {'MlFeatureValueMetric'} + excluded_fields = {"MlFeatureValueMetric"} for stream in self.streams_to_test(): with self.subTest(stream=stream): - found_catalog_names = {catalog['tap_stream_id'] for catalog in AllFieldsTest.found_catalogs} + found_catalog_names = { + catalog["tap_stream_id"] for catalog in AllFieldsTest.found_catalogs + } self.assertTrue(self.streams_to_test().issubset(found_catalog_names)) LOGGER.info("discovered schemas are OK") - expected_non_custom_fields = self.selected_fields.get(stream,set()) - excluded_fields + expected_non_custom_fields = ( + self.selected_fields.get(stream, set()) - excluded_fields + ) replicated_non_custom_fields = self.actual_fields.get(stream, set()) - #Verify at least one non-custom field is replicated - self.assertGreater(len(replicated_non_custom_fields),0, - msg = f"Replication didn't return any non-custom fields for stream {stream}") - - #verify that all the non_custom fields are replicated - self.assertEqual(replicated_non_custom_fields, expected_non_custom_fields, - msg = f"All non_custom fields are not no replicated for stream {stream}") - - #verify that automatic fields are also replicated along with non_custom_fields - self.assertTrue(self.expected_automatic_fields(stream).issubset(replicated_non_custom_fields), - msg = f"Automatic fields are not replicated for stream {stream}") - - #Verify custom fields are not replicated by checking the field name - num_custom, _ = self.count_custom_non_custom_fields(replicated_non_custom_fields) - self.assertEqual(num_custom, 0, - msg = f"Replicated some fields that are custom fields for stream {stream}") + # Verify at least one non-custom field is replicated + self.assertGreater( + len(replicated_non_custom_fields), + 0, + msg=f"Replication didn't return any non-custom fields for stream {stream}", + ) + + # verify that all the non_custom fields are replicated + self.assertEqual( + replicated_non_custom_fields, + expected_non_custom_fields, + msg=f"All non_custom fields are not no replicated for stream {stream}", + ) + + # verify that automatic fields are also replicated along with non_custom_fields + self.assertTrue( + self.expected_automatic_fields(stream).issubset( + replicated_non_custom_fields + ), + msg=f"Automatic fields are not replicated for stream {stream}", + ) + + # Verify custom fields are not replicated by checking the field name + num_custom, _ = self.count_custom_non_custom_fields( + replicated_non_custom_fields + ) + self.assertEqual( + num_custom, + 0, + msg=f"Replicated some fields that are custom fields for stream {stream}", + ) diff --git a/tests/test_salesforce_api_per_run_quota.py b/tests/test_salesforce_api_per_run_quota.py index 57ccee20..44e2a3da 100644 --- a/tests/test_salesforce_api_per_run_quota.py +++ b/tests/test_salesforce_api_per_run_quota.py @@ -1,13 +1,12 @@ from sfbase import SFBaseTest -from tap_tester import connections, runner, menagerie -from tap_tester.logger import LOGGER +from tap_tester import connections class SFAPIQuota(SFBaseTest): """ https://jira.talendforge.org/browse/TDL-23431 - This testcase makes sure we are able to configure the tap with specific - per_run quota for apis and verifies if tap is working as per the + This testcase makes sure we are able to configure the tap with specific + per_run quota for apis and verifies if tap is working as per the api quota limit set. """ @@ -17,22 +16,22 @@ class SFAPIQuota(SFBaseTest): some streams are excluded as they don't have any data """ - start_date = '2000-11-23T00:00:00Z' - per_run_quota = '1' + start_date = "2000-11-23T00:00:00Z" + per_run_quota = "1" streams_to_exclude = { - 'DatacloudAddress', - 'DatacloudCompany', - 'DatacloudContact', - 'DatacloudDandBCompany', - 'DatacloudOwnedEntity', - 'DatacloudPurchaseUsage', - 'FieldSecurityClassification', - 'ServiceAppointmentStatus', - 'WorkOrderLineItemStatus', - 'WorkOrderStatus', - 'ShiftStatus', - 'WorkStepStatus', - } + "DatacloudAddress", + "DatacloudCompany", + "DatacloudContact", + "DatacloudDandBCompany", + "DatacloudOwnedEntity", + "DatacloudPurchaseUsage", + "FieldSecurityClassification", + "ServiceAppointmentStatus", + "WorkOrderLineItemStatus", + "WorkOrderStatus", + "ShiftStatus", + "WorkStepStatus", + } @staticmethod def name(): @@ -52,24 +51,28 @@ def test_api_per_run_quota(self): For the sync mode, we have a higher total quota set, so it is unlikely to hit the total quota. Noticed that per run quota limit reaches only during the sync mode. """ expected_per_run_error = "Terminating replication due to allotted quota" - expected_total_quota_error = "Terminating replication to not continue past configured percentage" + expected_total_quota_error = ( + "Terminating replication to not continue past configured percentage" + ) conn_id = connections.ensure_connection(self) # Run a check job using orchestrator (discovery) try: - found_catalogs = self.run_and_verify_check_mode(conn_id) + found_catalogs = self.run_and_verify_check_mode(conn_id) except Exception as ex: self.assertIn(expected_per_run_error, str(ex.exception)) # table and field selection - test_catalogs = [catalog for catalog in found_catalogs - if catalog.get('stream_name') in self.streams_to_test()] + test_catalogs = [ + catalog + for catalog in found_catalogs + if catalog.get("stream_name") in self.streams_to_test() + ] # non_selected_fields are none self.perform_and_verify_table_and_field_selection(conn_id, test_catalogs) with self.assertRaises(Exception) as ex: - record_count_by_stream = self.run_and_verify_sync_mode(conn_id) + record_count_by_stream = self.run_and_verify_sync_mode(conn_id) self.assertIn(expected_per_run_error, str(ex.exception)) - diff --git a/tests/test_salesforce_api_total_quota.py b/tests/test_salesforce_api_total_quota.py index 5ff23c00..158a0cdc 100644 --- a/tests/test_salesforce_api_total_quota.py +++ b/tests/test_salesforce_api_total_quota.py @@ -1,5 +1,5 @@ from sfbase import SFBaseTest -from tap_tester import runner, menagerie, LOGGER, connections +from tap_tester import connections class SFAPIQuota(SFBaseTest): @@ -16,22 +16,22 @@ class SFAPIQuota(SFBaseTest): some streams are excluded as they don't have any data """ - start_date = '2000-11-23T00:00:00Z' - total_quota = '1' + start_date = "2000-11-23T00:00:00Z" + total_quota = "1" streams_to_exclude = { - 'DatacloudAddress', - 'DatacloudCompany', - 'DatacloudContact', - 'DatacloudDandBCompany', - 'DatacloudOwnedEntity', - 'DatacloudPurchaseUsage', - 'FieldSecurityClassification', - 'ServiceAppointmentStatus', - 'WorkOrderLineItemStatus', - 'WorkOrderStatus', - 'ShiftStatus', - 'WorkStepStatus', - } + "DatacloudAddress", + "DatacloudCompany", + "DatacloudContact", + "DatacloudDandBCompany", + "DatacloudOwnedEntity", + "DatacloudPurchaseUsage", + "FieldSecurityClassification", + "ServiceAppointmentStatus", + "WorkOrderLineItemStatus", + "WorkOrderStatus", + "ShiftStatus", + "WorkStepStatus", + } @staticmethod def name(): @@ -42,7 +42,7 @@ def streams_to_test(self): def test_api_total_quota(self): """ - Run the tap in check mode and verify it returns the error for quota limit reached. + Run the tap in check mode and verify it returns the error for quota limit reached. """ expected_total_quota_error = "Terminating replication to not continue past configured percentage of 1.0% total quota" @@ -50,7 +50,6 @@ def test_api_total_quota(self): # Run a check job using orchestrator (discovery) with self.assertRaises(Exception) as ex: - check_job_name = self.run_and_verify_check_mode(conn_id) + check_job_name = self.run_and_verify_check_mode(conn_id) self.assertIn(expected_total_quota_error, str(ex.exception)) - diff --git a/tests/test_salesforce_automatic_fields_bulk.py b/tests/test_salesforce_automatic_fields_bulk.py index ef4353e2..65a07bea 100644 --- a/tests/test_salesforce_automatic_fields_bulk.py +++ b/tests/test_salesforce_automatic_fields_bulk.py @@ -1,10 +1,7 @@ """ Test that with no fields selected for a stream automatic fields are still replicated """ -import unittest -from datetime import datetime, timedelta -from tap_tester import runner, connections from test_salesforce_automatic_fields_rest import SalesforceAutomaticFields diff --git a/tests/test_salesforce_automatic_fields_rest.py b/tests/test_salesforce_automatic_fields_rest.py index 03db2da2..bf897f08 100644 --- a/tests/test_salesforce_automatic_fields_rest.py +++ b/tests/test_salesforce_automatic_fields_rest.py @@ -1,27 +1,26 @@ """ Test that with no fields selected for a stream automatic fields are still replicated """ -import unittest -from datetime import datetime, timedelta -from tap_tester import runner, connections +from datetime import datetime, timedelta from base import SalesforceBaseTest +from tap_tester import connections, runner class SalesforceAutomaticFields(SalesforceBaseTest): """Test that with no fields selected for a stream automatic fields are still replicated""" - start_date = (datetime.now() + timedelta(days=-1)).strftime("%Y-%m-%dT00:00:00Z") + start_date = (datetime.now() + timedelta(days=-1)).strftime("%Y-%m-%dT00:00:00Z") @staticmethod def expected_sync_streams(): return { - 'Account', - 'Contact', - 'Lead', - 'Opportunity', - 'User', + "Account", + "Contact", + "Lead", + "Opportunity", + "User", } def automatic_fields_test(self): @@ -44,11 +43,16 @@ def automatic_fields_test(self): found_catalogs = self.run_and_verify_check_mode(conn_id) # table and field selection - test_catalogs_automatic_fields = [catalog for catalog in found_catalogs - if catalog.get('stream_name') in expected_streams] + test_catalogs_automatic_fields = [ + catalog + for catalog in found_catalogs + if catalog.get("stream_name") in expected_streams + ] self.perform_and_verify_table_and_field_selection( - conn_id, test_catalogs_automatic_fields, select_all_fields=False, + conn_id, + test_catalogs_automatic_fields, + select_all_fields=False, ) # run initial sync @@ -57,20 +61,23 @@ def automatic_fields_test(self): for stream in expected_streams: with self.subTest(stream=stream): - # expected values expected_keys = self.expected_automatic_fields().get(stream) # collect actual values data = synced_records.get(stream) - record_messages_keys = [set(row['data'].keys()) for row in data['messages'] - if row['action'] == 'upsert'] - + record_messages_keys = [ + set(row["data"].keys()) + for row in data["messages"] + if row["action"] == "upsert" + ] # Verify that you get some records for each stream self.assertGreater( - record_count_by_stream.get(stream, -1), 0, - msg="The number of records is not over the stream max limit") + record_count_by_stream.get(stream, -1), + 0, + msg="The number of records is not over the stream max limit", + ) # Verify that only the automatic fields are sent to the target for actual_keys in record_messages_keys: @@ -80,7 +87,7 @@ def automatic_fields_test(self): class SalesforceAutomaticFieldsRest(SalesforceAutomaticFields): """Test that with no fields selected for a stream automatic fields are still replicated""" - salesforce_api = 'REST' + salesforce_api = "REST" @staticmethod def name(): diff --git a/tests/test_salesforce_bookmarks.py b/tests/test_salesforce_bookmarks.py index 558d707f..a428158f 100644 --- a/tests/test_salesforce_bookmarks.py +++ b/tests/test_salesforce_bookmarks.py @@ -1,10 +1,10 @@ -from tap_tester.base_suite_tests.bookmark_test import BookmarkTest from sfbase import SFBaseTest +from tap_tester.base_suite_tests.bookmark_test import BookmarkTest class SFBookmarkTest(BookmarkTest, SFBaseTest): + salesforce_api = "BULK" - salesforce_api = 'BULK' @staticmethod def name(): return "tt_sf_bookmarks" @@ -12,30 +12,39 @@ def name(): @staticmethod def streams_to_test(): return { - 'User', - 'Publisher', - 'AppDefinition', + "User", + "Publisher", + "AppDefinition", } - bookmark_format ="%Y-%m-%dT%H:%M:%S.%fZ" + bookmark_format = "%Y-%m-%dT%H:%M:%S.%fZ" initial_bookmarks = {} streams_replication_method = {} + def streams_replication_methods(self): - streams_to_set_rep_method = [catalog['tap_stream_id'] for catalog in BookmarkTest.test_catalogs - if 'forced-replication-method' not in catalog['metadata'].keys()] + streams_to_set_rep_method = [ + catalog["tap_stream_id"] + for catalog in BookmarkTest.test_catalogs + if "forced-replication-method" not in catalog["metadata"].keys() + ] if len(streams_to_set_rep_method) > 0: - self.streams_replication_method = {stream: 'INCREMENTAL' - for stream in streams_to_set_rep_method} + self.streams_replication_method = { + stream: "INCREMENTAL" for stream in streams_to_set_rep_method + } return self.streams_replication_method def adjusted_expected_replication_method(self): - streams_to_set_rep_method = [catalog['tap_stream_id'] for catalog in BookmarkTest.test_catalogs - if 'forced-replication-method' not in catalog['metadata'].keys()] + streams_to_set_rep_method = [ + catalog["tap_stream_id"] + for catalog in BookmarkTest.test_catalogs + if "forced-replication-method" not in catalog["metadata"].keys() + ] expected_replication_methods = self.expected_replication_method() if self.streams_replication_method: - for stream in streams_to_set_rep_method : - expected_replication_methods[stream] = self.streams_replication_method[stream] + for stream in streams_to_set_rep_method: + expected_replication_methods[stream] = self.streams_replication_method[ + stream + ] return expected_replication_methods return expected_replication_methods - diff --git a/tests/test_salesforce_discovery_bulk.py b/tests/test_salesforce_discovery_bulk.py index f27f2d33..a4ef92c7 100644 --- a/tests/test_salesforce_discovery_bulk.py +++ b/tests/test_salesforce_discovery_bulk.py @@ -1,6 +1,4 @@ """Test tap discovery mode and metadata/annotated-schema.""" -import unittest -from tap_tester import menagerie, connections from test_salesforce_discovery_rest import DiscoveryTest @@ -13,5 +11,5 @@ def name(): return "tt_salesforce_disco_bulk" def test_discovery(self): - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" self.discovery_test() diff --git a/tests/test_salesforce_discovery_rest.py b/tests/test_salesforce_discovery_rest.py index c78244b7..ba576581 100644 --- a/tests/test_salesforce_discovery_rest.py +++ b/tests/test_salesforce_discovery_rest.py @@ -1,8 +1,7 @@ """Test tap discovery mode and metadata/annotated-schema.""" -import unittest -from tap_tester import menagerie, connections, LOGGER from base import SalesforceBaseTest +from tap_tester import LOGGER, connections, menagerie class DiscoveryTest(SalesforceBaseTest): @@ -28,9 +27,9 @@ def discovery_test(self): # BUG | https://jira.talendforge.org/browse/TDL-15748 # The following streams stopped being discovered 10/10/2021 # When bug is addressed fix the marked lines - missing_streams = {'DataAssetUsageTrackingInfo', 'DataAssetSemanticGraphEdge'} + missing_streams = {"DataAssetUsageTrackingInfo", "DataAssetSemanticGraphEdge"} - streams_to_test = self.expected_streams() - missing_streams # BUG_TDL-15748 + streams_to_test = self.expected_streams() - missing_streams # BUG_TDL-15748 # streams_to_test_prime = self.expected_streams().difference(self.get_unsupported_by_bulk_api()) # self.assertEqual(len(streams_to_test), len(streams_to_test_prime), msg="Expectations are invalid.") # BUG_TDL-15748 @@ -39,7 +38,7 @@ def discovery_test(self): found_catalogs = self.run_and_verify_check_mode(conn_id) # verify the tap only discovers the expected streams - found_catalog_names = {catalog['tap_stream_id'] for catalog in found_catalogs} + found_catalog_names = {catalog["tap_stream_id"] for catalog in found_catalogs} self.assertTrue(streams_to_test.issubset(found_catalog_names)) LOGGER.info("discovered schemas are OK") @@ -54,34 +53,57 @@ def discovery_test(self): for stream in streams_to_test: with self.subTest(stream=stream): - catalog = next(iter([catalog for catalog in found_catalogs - if catalog["stream_name"] == stream])) + catalog = next( + iter( + [ + catalog + for catalog in found_catalogs + if catalog["stream_name"] == stream + ] + ) + ) assert catalog # based on previous tests this should always be found # gather expectations expected_replication_keys = self.expected_replication_keys()[stream] expected_primary_keys = self.expected_primary_keys()[stream] expected_replication_method = self.expected_replication_method()[stream] - expected_automatic_fields = expected_primary_keys | expected_replication_keys + expected_automatic_fields = ( + expected_primary_keys | expected_replication_keys + ) # gather results - schema_and_metadata = menagerie.get_annotated_schema(conn_id, catalog['stream_id']) + schema_and_metadata = menagerie.get_annotated_schema( + conn_id, catalog["stream_id"] + ) metadata = schema_and_metadata["metadata"] schema = schema_and_metadata["annotated-schema"] - stream_properties = [item for item in metadata if item.get("breadcrumb") == []] - actual_replication_keys = set(stream_properties[0].get( - "metadata", {self.REPLICATION_KEYS: []}).get(self.REPLICATION_KEYS, []) + stream_properties = [ + item for item in metadata if item.get("breadcrumb") == [] + ] + actual_replication_keys = set( + stream_properties[0] + .get("metadata", {self.REPLICATION_KEYS: []}) + .get(self.REPLICATION_KEYS, []) ) - actual_primary_keys = set(stream_properties[0].get( - "metadata", {self.PRIMARY_KEYS: []}).get(self.PRIMARY_KEYS, []) + actual_primary_keys = set( + stream_properties[0] + .get("metadata", {self.PRIMARY_KEYS: []}) + .get(self.PRIMARY_KEYS, []) + ) + actual_replication_method = ( + stream_properties[0] + .get("metadata", {self.REPLICATION_METHOD: None}) + .get(self.REPLICATION_METHOD) ) - actual_replication_method = stream_properties[0].get( - "metadata", {self.REPLICATION_METHOD: None}).get(self.REPLICATION_METHOD) - # verify there is only 1 top level breadcrumb - self.assertTrue(len(stream_properties) == 1, - msg="There is NOT only one top level breadcrumb for {}".format(stream) + \ - "\nstream_properties | {}".format(stream_properties)) + self.assertTrue( + len(stream_properties) == 1, + msg="There is NOT only one top level breadcrumb for {}".format( + stream + ) + + "\nstream_properties | {}".format(stream_properties), + ) # verify replication key(s) self.assertSetEqual(expected_replication_keys, actual_replication_keys) @@ -92,7 +114,9 @@ def discovery_test(self): # BUG_TDL-9711 | https://jira.talendforge.org/browse/TDL-9711 # [tap-salesforce] discovered streams have unexpected replication methods # all values are set to None - LOGGER.warning("Skipping 'expected replication method' asssertions for %s", stream) + LOGGER.warning( + "Skipping 'expected replication method' asssertions for %s", stream + ) # verify the actual replication matches our expected replication method # self.assertEqual(expected_replication_method, actual_replication_method) # BUG_TDL-9711 @@ -104,56 +128,102 @@ def discovery_test(self): # verify that primary, replication and foreign keys # are given the inclusion of automatic in annotated schema. - actual_automatic_fields = {key for key, value in schema["properties"].items() - if value.get("inclusion") == "automatic"} + actual_automatic_fields = { + key + for key, value in schema["properties"].items() + if value.get("inclusion") == "automatic" + } self.assertEqual(expected_automatic_fields, actual_automatic_fields) # verify that primary, replication and foreign keys # are given the inclusion of automatic in metadata. - actual_automatic_fields = {item.get("breadcrumb", ["properties", None])[1] - for item in metadata - if item.get("metadata").get("inclusion") == "automatic"} - self.assertEqual(expected_automatic_fields, - actual_automatic_fields, - msg="expected {} automatic fields but got {}".format( - expected_automatic_fields, - actual_automatic_fields)) + actual_automatic_fields = { + item.get("breadcrumb", ["properties", None])[1] + for item in metadata + if item.get("metadata").get("inclusion") == "automatic" + } + self.assertEqual( + expected_automatic_fields, + actual_automatic_fields, + msg="expected {} automatic fields but got {}".format( + expected_automatic_fields, actual_automatic_fields + ), + ) # TODO |https://jira.talendforge.org/browse/TDL-17122 # Add test case for unsupported json object fields skip_available_streams = { - 'PermissionSetEventStore', # rest - 'CartDeliveryGroup', # bulk - 'WebCart', # bulk + "PermissionSetEventStore", # rest + "CartDeliveryGroup", # bulk + "WebCart", # bulk } # BUG_TDL-9816 | https://jira.talendforge.org/browse/TDL-9816 # [tap-salesforce] discovered streams missing `inlcusion` values failing_available_streams = { - 'StaticResource', 'Contract', 'ContentVersion', 'Scontrol', - 'MobileApplicationDetail', 'EntityDefinition', 'User', 'Contact', - 'Attachment', 'ContactPointAddress', 'MailmergeTemplate', 'Lead', 'Order', - 'AlternativePaymentMethod', 'CardPaymentMethod', 'ServiceContract', - 'ServiceTerritoryMember', 'ReportEvent', 'EmailCapture', 'ListViewEvent', - 'WorkOrderLineItem', 'LeadCleanInfo', 'ServiceTerritory', 'DigitalWallet', - 'ApiEvent', 'WorkOrder', 'ContactCleanInfo', 'ResourceAbsence', 'ReturnOrder', - 'LegalEntity', 'PaymentMethod', 'EventLogFile', 'ServiceAppointment', - 'DandBCompany', 'AccountCleanInfo', 'Organization', 'Document', 'Account', - 'Address', 'FulfillmentOrder', 'Asset' + "StaticResource", + "Contract", + "ContentVersion", + "Scontrol", + "MobileApplicationDetail", + "EntityDefinition", + "User", + "Contact", + "Attachment", + "ContactPointAddress", + "MailmergeTemplate", + "Lead", + "Order", + "AlternativePaymentMethod", + "CardPaymentMethod", + "ServiceContract", + "ServiceTerritoryMember", + "ReportEvent", + "EmailCapture", + "ListViewEvent", + "WorkOrderLineItem", + "LeadCleanInfo", + "ServiceTerritory", + "DigitalWallet", + "ApiEvent", + "WorkOrder", + "ContactCleanInfo", + "ResourceAbsence", + "ReturnOrder", + "LegalEntity", + "PaymentMethod", + "EventLogFile", + "ServiceAppointment", + "DandBCompany", + "AccountCleanInfo", + "Organization", + "Document", + "Account", + "Address", + "FulfillmentOrder", + "Asset", } # verify that all other fields have inclusion of available # This assumes there are no unsupported fields for SaaS sources if stream in failing_available_streams.union(skip_available_streams): - LOGGER.warning("Skipping 'metadata inclusion available' asssertion for %s", stream) - else: # BUG_TDL-9816 comment to reproduce + LOGGER.warning( + "Skipping 'metadata inclusion available' asssertion for %s", + stream, + ) + else: # BUG_TDL-9816 comment to reproduce self.assertTrue( - all({item.get("metadata").get("inclusion") == "available" - for item in metadata - if item.get("breadcrumb", []) != [] - and item.get("breadcrumb", ["properties", None])[1] - not in actual_automatic_fields}), - msg="Not all non key properties are set to available in metadata") + all( + { + item.get("metadata").get("inclusion") == "available" + for item in metadata + if item.get("breadcrumb", []) != [] + and item.get("breadcrumb", ["properties", None])[1] + not in actual_automatic_fields + } + ), + msg="Not all non key properties are set to available in metadata", + ) class DiscoveryTestRest(DiscoveryTest): @@ -164,5 +234,5 @@ def name(): return "tt_salesforce_disco_rest" def test_discovery(self): - self.salesforce_api = 'REST' + self.salesforce_api = "REST" self.discovery_test() diff --git a/tests/test_salesforce_full_table_replication.py b/tests/test_salesforce_full_table_replication.py index 2e71cc59..fb92ed60 100644 --- a/tests/test_salesforce_full_table_replication.py +++ b/tests/test_salesforce_full_table_replication.py @@ -1,11 +1,12 @@ """ Test tap gets all records for streams with full replication """ -import json -from tap_tester import menagerie, runner, connections +import json from base import SalesforceBaseTest +from tap_tester import connections, menagerie, runner + class SalesforceFullReplicationTest(SalesforceBaseTest): """Test tap gets all records for streams with full replication""" @@ -24,7 +25,7 @@ def test_run(self): For EACH stream that is fully replicated there are multiple rows of data with """ - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" conn_id = connections.ensure_connection(self) self.conn_id = conn_id @@ -32,15 +33,21 @@ def test_run(self): found_catalogs = self.run_and_verify_check_mode(conn_id) # Test for streams that has the replication method as Full - full_streams = {'TabDefinition', - # Disabled on 09/03/25 - # 'FormulaFunctionAllowedType', - # 'FormulaFunction' - } - - our_catalogs = [catalog for catalog in found_catalogs if - catalog.get('tap_stream_id') in full_streams] - self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) + full_streams = { + "TabDefinition", + # Disabled on 09/03/25 + # 'FormulaFunctionAllowedType', + # 'FormulaFunction' + } + + our_catalogs = [ + catalog + for catalog in found_catalogs + if catalog.get("tap_stream_id") in full_streams + ] + self.select_all_streams_and_fields( + conn_id, our_catalogs, select_all_fields=True + ) # Run a sync job using orchestrator first_sync_record_count = self.run_and_verify_sync(conn_id) @@ -62,28 +69,43 @@ def test_run(self): for stream in full_streams.difference(self.child_streams()): with self.subTest(stream=stream): - # verify that there is more than 1 record of data - setup necessary - self.assertGreater(first_sync_record_count.get(stream, 0), 1, - msg="Data isn't set up to be able to test full sync") + self.assertGreater( + first_sync_record_count.get(stream, 0), + 1, + msg="Data isn't set up to be able to test full sync", + ) # verify that you get the same or more data the 2nd time around self.assertGreaterEqual( second_sync_record_count.get(stream, 0), first_sync_record_count.get(stream, 0), - msg="second syc didn't have more records, full sync not verified") + msg="second syc didn't have more records, full sync not verified", + ) # Verify if the bookmarks for first and second sync for full table are None - self.assertIsNone( first_sync_state['bookmarks'].get(stream).get('version')) - self.assertIsNone( second_sync_state['bookmarks'].get(stream).get('version')) + self.assertIsNone( + first_sync_state["bookmarks"].get(stream).get("version") + ) + self.assertIsNone( + second_sync_state["bookmarks"].get(stream).get("version") + ) # verify all data from 1st sync included in 2nd sync - first_data = [record["data"] for record - in first_sync_records.get(stream, {}).get("messages", {"data": {}}) - if record["action"] == 'upsert'] - second_data = [record["data"] for record - in second_sync_records.get(stream, {}).get("messages", {"data": {}}) - if record["action"] == 'upsert'] + first_data = [ + record["data"] + for record in first_sync_records.get(stream, {}).get( + "messages", {"data": {}} + ) + if record["action"] == "upsert" + ] + second_data = [ + record["data"] + for record in second_sync_records.get(stream, {}).get( + "messages", {"data": {}} + ) + if record["action"] == "upsert" + ] same_records = 0 for first_record in first_data: @@ -97,5 +119,8 @@ def test_run(self): same_records += 1 break - self.assertEqual(len(first_data), same_records, - msg="Not all data from the first sync was in the second sync") + self.assertEqual( + len(first_data), + same_records, + msg="Not all data from the first sync was in the second sync", + ) diff --git a/tests/test_salesforce_incremental_table_reset.py b/tests/test_salesforce_incremental_table_reset.py index eca6d9c4..ceef313d 100644 --- a/tests/test_salesforce_incremental_table_reset.py +++ b/tests/test_salesforce_incremental_table_reset.py @@ -1,4 +1,3 @@ -import copy from sfbase import SFBaseTest from tap_tester.base_suite_tests.table_reset_test import TableResetTest @@ -12,11 +11,11 @@ def name(): return "tt_sf_table_reset" def streams_to_test(self): - return ({'Account', 'Contact', 'User'}) + return {"Account", "Contact", "User"} @property def reset_stream(self): - return ('User') + return "User" def manipulate_state(self, current_state): # no state manipulation needed for this tap diff --git a/tests/test_salesforce_lookback_window.py b/tests/test_salesforce_lookback_window.py index 915a2b06..49632f48 100644 --- a/tests/test_salesforce_lookback_window.py +++ b/tests/test_salesforce_lookback_window.py @@ -1,9 +1,10 @@ from datetime import datetime, timedelta -from tap_tester import connections, runner, menagerie + from base import SalesforceBaseTest +from tap_tester import connections, menagerie, runner -class SalesforceLookbackWindow(SalesforceBaseTest): +class SalesforceLookbackWindow(SalesforceBaseTest): # subtract the desired amount of seconds form the date and return def get_simulated_date(self, dtime, format, seconds=0): date_stripped = datetime.strptime(dtime, format) @@ -13,33 +14,27 @@ def get_simulated_date(self, dtime, format, seconds=0): @staticmethod def name(): - return 'tap_tester_salesforce_lookback_window' + return "tap_tester_salesforce_lookback_window" def get_properties(self): # pylint: disable=arguments-differ return { - 'start_date' : '2021-11-10T00:00:00Z', - 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', - 'select_fields_by_default': 'true', - 'api_type': self.salesforce_api, - 'is_sandbox': 'false', - 'lookback_window': 86400 + "start_date": "2021-11-10T00:00:00Z", + "instance_url": "https://singer2-dev-ed.my.salesforce.com", + "select_fields_by_default": "true", + "api_type": self.salesforce_api, + "is_sandbox": "false", + "lookback_window": 86400, } def expected_sync_streams(self): - return { - 'Account' - } + return {"Account"} def run_test(self): # create connection conn_id = connections.ensure_connection(self) # create state file state = { - 'bookmarks':{ - 'Account': { - 'SystemModstamp': '2021-11-12T00:00:00.000000Z' - } - } + "bookmarks": {"Account": {"SystemModstamp": "2021-11-12T00:00:00.000000Z"}} } # set state file to run in sync mode menagerie.set_state(conn_id, state) @@ -49,14 +44,19 @@ def run_test(self): # select certain catalogs expected_streams = self.expected_sync_streams() - catalog_entries = [catalog for catalog in found_catalogs - if catalog.get('tap_stream_id') in expected_streams] + catalog_entries = [ + catalog + for catalog in found_catalogs + if catalog.get("tap_stream_id") in expected_streams + ] # stream and field selection self.select_all_streams_and_fields(conn_id, catalog_entries) # make 'Account' stream as INCREMENTAL to use lookback window - self.set_replication_methods(conn_id, catalog_entries, {'Account': self.INCREMENTAL}) + self.set_replication_methods( + conn_id, catalog_entries, {"Account": self.INCREMENTAL} + ) # run sync self.run_and_verify_sync(conn_id) @@ -68,40 +68,58 @@ def run_test(self): expected_replication_keys = self.expected_replication_keys() # get bookmark ie. date from which the sync started - bookmark = state.get('bookmarks').get('Account').get('SystemModstamp') + bookmark = state.get("bookmarks").get("Account").get("SystemModstamp") # calculate the simulated bookmark by subtracting lookback window seconds - bookmark_with_lookback_window = self.get_simulated_date(bookmark, format=self.BOOKMARK_COMPARISON_FORMAT, seconds=self.get_properties()['lookback_window']) + bookmark_with_lookback_window = self.get_simulated_date( + bookmark, + format=self.BOOKMARK_COMPARISON_FORMAT, + seconds=self.get_properties()["lookback_window"], + ) for stream in expected_streams: with self.subTest(stream=stream): - # get replication key for stream replication_key = list(expected_replication_keys[stream])[0] # get records - records = [record.get('data') for record in sync_records.get(stream).get('messages') - if record.get('action') == 'upsert'] + records = [ + record.get("data") + for record in sync_records.get(stream).get("messages") + if record.get("action") == "upsert" + ] # verify if we get records in ASCENDING order: # every record's date should be lesser than the next record's date for i in range(len(records) - 1): - self.assertLessEqual(self.parse_date(records[i].get(replication_key)), self.parse_date(records[i+1].get(replication_key))) + self.assertLessEqual( + self.parse_date(records[i].get(replication_key)), + self.parse_date(records[i + 1].get(replication_key)), + ) # Verify the sync records respect the (simulated) bookmark value for record in records: - self.assertGreaterEqual(self.parse_date(record.get(replication_key)), self.parse_date(bookmark_with_lookback_window), - msg='The record does not respect the lookback window.') + self.assertGreaterEqual( + self.parse_date(record.get(replication_key)), + self.parse_date(bookmark_with_lookback_window), + msg="The record does not respect the lookback window.", + ) # check if the 1st record is between lookback date and bookmark: # lookback_date <= record < bookmark (state file date when sync started) - self.assertLessEqual(self.parse_date(bookmark_with_lookback_window), self.parse_date(records[0].get(replication_key))) - self.assertLess(self.parse_date(records[0].get(replication_key)), self.parse_date(bookmark)) + self.assertLessEqual( + self.parse_date(bookmark_with_lookback_window), + self.parse_date(records[0].get(replication_key)), + ) + self.assertLess( + self.parse_date(records[0].get(replication_key)), + self.parse_date(bookmark), + ) def test_run(self): # run with REST API - self.salesforce_api = 'REST' + self.salesforce_api = "REST" self.run_test() # run with BULK API - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" self.run_test() diff --git a/tests/test_salesforce_select_by_default.py b/tests/test_salesforce_select_by_default.py index 6140fcc1..f5b84552 100644 --- a/tests/test_salesforce_select_by_default.py +++ b/tests/test_salesforce_select_by_default.py @@ -1,6 +1,5 @@ -from tap_tester import connections, runner, menagerie - from sfbase import SFBaseTest +from tap_tester import connections, runner class SFSelectByDefault(SFBaseTest): @@ -11,12 +10,12 @@ def name(): @staticmethod def streams_to_test(): return { - 'Account', - 'LoginGeo', + "Account", + "LoginGeo", } def setUp(self): - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" self.start_date = "2021-11-11T00:00:00Z" # instantiate connection SFSelectByDefault.conn_id = connections.ensure_connection(self) @@ -25,9 +24,12 @@ def setUp(self): SFSelectByDefault.found_catalogs = self.run_and_verify_check_mode(self.conn_id) # table and field selection - test_catalogs = [catalog for catalog in self.found_catalogs - if catalog.get('tap_stream_id') in self.streams_to_test()] - + test_catalogs = [ + catalog + for catalog in self.found_catalogs + if catalog.get("tap_stream_id") in self.streams_to_test() + ] + SFSelectByDefault.test_streams = self.streams_to_test() self.perform_and_verify_table_selection(self.conn_id, test_catalogs) @@ -42,12 +44,17 @@ def test_no_unexpected_streams_replicated(self): self.assertSetEqual(synced_stream_names, self.test_streams) def test_default_fields_for_streams_are_replicated(self): - expected_rep_keys = self.get_select_by_default_fields(self.found_catalogs, self.conn_id) + expected_rep_keys = self.get_select_by_default_fields( + self.found_catalogs, self.conn_id + ) for stream in self.test_streams: with self.subTest(stream=stream): - # gather results + # gather results fields_replicated = self.actual_fields.get(stream, set()) # verify that all fields are sent to the target # test the combination of all records - self.assertSetEqual(fields_replicated, expected_rep_keys[stream], - logging=f"verify all fields are replicated for stream {stream}") + self.assertSetEqual( + fields_replicated, + expected_rep_keys[stream], + logging=f"verify all fields are replicated for stream {stream}", + ) diff --git a/tests/test_salesforce_start_date.py b/tests/test_salesforce_start_date.py index 08a5f2fb..521d5977 100644 --- a/tests/test_salesforce_start_date.py +++ b/tests/test_salesforce_start_date.py @@ -1,11 +1,10 @@ -import unittest -from tap_tester import connections, runner, LOGGER - from base import SalesforceBaseTest +from tap_tester import LOGGER, connections, runner class SalesforceStartDateTest(SalesforceBaseTest): """Test that core objects do not obey the start date""" + start_date_1 = "" start_date_2 = "" @@ -16,16 +15,16 @@ def name(): @staticmethod def expected_sync_streams(): return { - 'Account', # "2021-11-11T03:50:52.000000Z" - 'Contact', # "2021-11-11T03:50:52.000000Z" - 'Lead', # "2021-11-23T11:48:24.000000Z" - 'Opportunity', # "2021-11-11T03:50:52.000000Z" - 'User', # "2021-11-23T11:48:24.000000Z" + "Account", # "2021-11-11T03:50:52.000000Z" + "Contact", # "2021-11-11T03:50:52.000000Z" + "Lead", # "2021-11-23T11:48:24.000000Z" + "Opportunity", # "2021-11-11T03:50:52.000000Z" + "User", # "2021-11-23T11:48:24.000000Z" } def test_run(self): """Instantiate start date according to the desired data set and run the test""" - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" self.assertTrue(self.expected_sync_streams().issubset(self.expected_streams())) @@ -45,10 +44,14 @@ def test_run(self): found_catalogs_1 = self.run_and_verify_check_mode(conn_id_1) # table and field selection - test_catalogs_1_all_fields = [catalog for catalog in found_catalogs_1 - if catalog.get('tap_stream_id') in self.expected_sync_streams()] - self.perform_and_verify_table_and_field_selection(conn_id_1, test_catalogs_1_all_fields, - select_all_fields=True) + test_catalogs_1_all_fields = [ + catalog + for catalog in found_catalogs_1 + if catalog.get("tap_stream_id") in self.expected_sync_streams() + ] + self.perform_and_verify_table_and_field_selection( + conn_id_1, test_catalogs_1_all_fields, select_all_fields=True + ) # run initial sync record_count_by_stream_1 = self.run_and_verify_sync(conn_id_1) @@ -58,7 +61,11 @@ def test_run(self): ### Update START DATE Between Sync3s ########################################################################## - LOGGER.info("REPLICATION START DATE CHANGE: %s ===>>> %s", self.start_date, self.start_date_2) + LOGGER.info( + "REPLICATION START DATE CHANGE: %s ===>>> %s", + self.start_date, + self.start_date_2, + ) self.start_date = self.start_date_2 ########################################################################## @@ -72,15 +79,22 @@ def test_run(self): found_catalogs_2 = self.run_and_verify_check_mode(conn_id_2) # table and field selection - test_catalogs_2_all_fields = [catalog for catalog in found_catalogs_2 - if catalog.get('tap_stream_id') in self.expected_sync_streams()] - self.perform_and_verify_table_and_field_selection(conn_id_2, test_catalogs_2_all_fields, select_all_fields=True) + test_catalogs_2_all_fields = [ + catalog + for catalog in found_catalogs_2 + if catalog.get("tap_stream_id") in self.expected_sync_streams() + ] + self.perform_and_verify_table_and_field_selection( + conn_id_2, test_catalogs_2_all_fields, select_all_fields=True + ) # run sync record_count_by_stream_2 = self.run_and_verify_sync(conn_id_2) replicated_row_count_2 = sum(record_count_by_stream_2.values()) - self.assertGreater(replicated_row_count_2, 0, msg="failed to replicate any data") + self.assertGreater( + replicated_row_count_2, 0, msg="failed to replicate any data" + ) LOGGER.info("total replicated row count: %s", replicated_row_count_2) synced_records_2 = runner.get_records_from_target_output() diff --git a/tests/test_salesforce_switch_rep_method_ft_incrmntl.py b/tests/test_salesforce_switch_rep_method_ft_incrmntl.py index ebab4a9d..87c81435 100644 --- a/tests/test_salesforce_switch_rep_method_ft_incrmntl.py +++ b/tests/test_salesforce_switch_rep_method_ft_incrmntl.py @@ -1,21 +1,25 @@ -from tap_tester import runner, menagerie, connections from sfbase import SFBaseTest +from tap_tester import connections, menagerie, runner class SFSwitchRepMethodIncrmntl(SFBaseTest): + start_date = "2000-11-11T00:00:00Z" # to get max data available for testing - start_date = '2000-11-11T00:00:00Z'#to get max data available for testing @staticmethod def name(): return "tt_sf_table_switch_rep_method_ft_incrmntl" def expected_sync_streams(self): - streams = self.switchable_streams() - {'FlowDefinitionView','EntityDefinition', 'EventLogFile'} + streams = self.switchable_streams() - { + "FlowDefinitionView", + "EntityDefinition", + "EventLogFile", + } # Excluded the above two streams due to the bug TDL-24514 return self.partition_streams(streams) def test_run(self): - self.salesforce_api = 'BULK' + self.salesforce_api = "BULK" replication_keys = self.expected_replication_keys() primary_keys = self.expected_primary_keys() @@ -27,11 +31,16 @@ def test_run(self): # Select only the expected streams tables expected_streams = self.expected_sync_streams() - catalog_entries = [ce for ce in found_catalogs if ce['tap_stream_id'] in expected_streams] + catalog_entries = [ + ce for ce in found_catalogs if ce["tap_stream_id"] in expected_streams + ] self.select_all_streams_and_fields(conn_id, catalog_entries) - streams_replication_methods = {stream: self.FULL_TABLE - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) + streams_replication_methods = { + stream: self.FULL_TABLE for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) # Run a sync job using orchestrator fulltbl_sync_record_count = self.run_and_verify_sync_mode(conn_id) @@ -39,10 +48,13 @@ def test_run(self): fulltbl_sync_bookmarks = menagerie.get_state(conn_id) - #Switch the replication method from full table to Incremental - streams_replication_methods = {stream: self.INCREMENTAL - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) + # Switch the replication method from full table to Incremental + streams_replication_methods = { + stream: self.INCREMENTAL for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) # SYNC 2 incrmntl_sync_record_count = self.run_and_verify_sync_mode(conn_id) @@ -58,50 +70,76 @@ def test_run(self): replication_key = list(replication_keys[stream])[0] # Verify at least 1 record was replicated in the Incrmental sync - self.assertGreater(incrmntl_sync_count, 0, - msg="We are not fully testing bookmarking for {}".format(stream)) + self.assertGreater( + incrmntl_sync_count, + 0, + msg="We are not fully testing bookmarking for {}".format(stream), + ) # data from record messages """ If implementing in tap-tester framework the primary key implementation should account for compound primary keys """ - self.assertEqual(1, len(list(primary_keys[stream])), - msg="Compound primary keys require a change to test expectations") + self.assertEqual( + 1, + len(list(primary_keys[stream])), + msg="Compound primary keys require a change to test expectations", + ) primary_key = list(primary_keys[stream])[0] - fulltbl_sync_messages = [record['data'] for record in - fulltbl_sync_records.get(stream, {}).get('messages') - if record.get('action') == 'upsert'] - filtered_fulltbl_sync_messages = [message for message in fulltbl_sync_messages - if message[replication_key] >= self.start_date] - fulltbl_primary_keys = {message[primary_key] for message in filtered_fulltbl_sync_messages} - incrmntl_sync_messages = [record['data'] for record in - incrmntl_sync_records.get(stream, {}).get('messages') - if record.get('action') == 'upsert'] - incrmntl_primary_keys = {message[primary_key] for message in incrmntl_sync_messages} - - #Verify all records are synced in the second sync + fulltbl_sync_messages = [ + record["data"] + for record in fulltbl_sync_records.get(stream, {}).get("messages") + if record.get("action") == "upsert" + ] + filtered_fulltbl_sync_messages = [ + message + for message in fulltbl_sync_messages + if message[replication_key] >= self.start_date + ] + fulltbl_primary_keys = { + message[primary_key] for message in filtered_fulltbl_sync_messages + } + incrmntl_sync_messages = [ + record["data"] + for record in incrmntl_sync_records.get(stream, {}).get("messages") + if record.get("action") == "upsert" + ] + incrmntl_primary_keys = { + message[primary_key] for message in incrmntl_sync_messages + } + + # Verify all records are synced in the second sync self.assertTrue(fulltbl_primary_keys.issubset(incrmntl_primary_keys)) """ Modify the the activate version assertion accordingly based on the outcome of BUG #TDL-24467 if needed """ - #verify that the last message is not a activate version message for incremental sync - self.assertNotEqual('activate_version', incrmntl_sync_records[stream]['messages'][-1]['action']) - - #verify that the table version incremented after every sync - self.assertGreater(incrmntl_sync_records[stream]['table_version'], - fulltbl_sync_records[stream]['table_version'], - msg = "Table version is not incremented after a successful sync") + # verify that the last message is not a activate version message for incremental sync + self.assertNotEqual( + "activate_version", + incrmntl_sync_records[stream]["messages"][-1]["action"], + ) + + # verify that the table version incremented after every sync + self.assertGreater( + incrmntl_sync_records[stream]["table_version"], + fulltbl_sync_records[stream]["table_version"], + msg="Table version is not incremented after a successful sync", + ) # bookmarked states (top level objects) - incrmntl_bookmark_key_value = incrmntl_sync_bookmarks.get('bookmarks', {}).get(stream) + incrmntl_bookmark_key_value = incrmntl_sync_bookmarks.get( + "bookmarks", {} + ).get(stream) # bookmarked states (actual values) - incrmntl_bookmark_value = incrmntl_bookmark_key_value.get(replication_key) + incrmntl_bookmark_value = incrmntl_bookmark_key_value.get( + replication_key + ) # Verify the incremental sync sets a bookmark of the expected form self.assertIsNotNone(incrmntl_bookmark_key_value) - #verify that bookmarks are present after switching to Incremental rep method + # verify that bookmarks are present after switching to Incremental rep method self.assertIsNotNone(incrmntl_bookmark_value) diff --git a/tests/test_salesforce_switch_rep_method_incrmntl_ft.py b/tests/test_salesforce_switch_rep_method_incrmntl_ft.py index 9b1d254a..fe1f455c 100644 --- a/tests/test_salesforce_switch_rep_method_incrmntl_ft.py +++ b/tests/test_salesforce_switch_rep_method_incrmntl_ft.py @@ -1,21 +1,25 @@ -from tap_tester import runner, menagerie, connections from sfbase import SFBaseTest +from tap_tester import connections, menagerie, runner class SFSwitchRepMethodFulltable(SFBaseTest): + start_date = "2000-01-23T00:00:00Z" - start_date = '2000-01-23T00:00:00Z' @staticmethod def name(): return "tt_sf_table_switch_rep_method_incrmntl_ft" def expected_sync_streams(self): - streams = self.switchable_streams() - {'FlowDefinitionView', 'EntityDefinition', 'EventLogFile'} + streams = self.switchable_streams() - { + "FlowDefinitionView", + "EntityDefinition", + "EventLogFile", + } # Excluded the above two streams due to the bug TDL-24514 return self.partition_streams(streams) def test_run(self): - self.salesforce_api = 'REST' + self.salesforce_api = "REST" replication_keys = self.expected_replication_keys() primary_keys = self.expected_primary_keys() # SYNC 1 @@ -26,11 +30,16 @@ def test_run(self): # Select only the expected streams tables expected_streams = self.expected_sync_streams() - catalog_entries = [ce for ce in found_catalogs if ce['tap_stream_id'] in expected_streams] + catalog_entries = [ + ce for ce in found_catalogs if ce["tap_stream_id"] in expected_streams + ] self.select_all_streams_and_fields(conn_id, catalog_entries) - streams_replication_methods = {stream: self.INCREMENTAL - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) + streams_replication_methods = { + stream: self.INCREMENTAL for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) # Run a sync job using orchestrator incrmntl_sync_record_count = self.run_and_verify_sync_mode(conn_id) @@ -38,10 +47,13 @@ def test_run(self): incrmntl_sync_bookmarks = menagerie.get_state(conn_id) - #Switch the replication method from incremental to full table - streams_replication_methods = {stream: self.FULL_TABLE - for stream in expected_streams} - self.set_replication_methods(conn_id, catalog_entries, streams_replication_methods) + # Switch the replication method from incremental to full table + streams_replication_methods = { + stream: self.FULL_TABLE for stream in expected_streams + } + self.set_replication_methods( + conn_id, catalog_entries, streams_replication_methods + ) # SYNC 2 fulltbl_sync_record_count = self.run_and_verify_sync_mode(conn_id) @@ -56,54 +68,80 @@ def test_run(self): fulltbl_sync_count = fulltbl_sync_record_count.get(stream, 0) replication_key = list(replication_keys[stream])[0] # Verify at least 1 record was replicated in the fulltbl sync - self.assertGreater(fulltbl_sync_count, 0, - msg="We are not fully testing bookmarking for {}".format(stream)) + self.assertGreater( + fulltbl_sync_count, + 0, + msg="We are not fully testing bookmarking for {}".format(stream), + ) # data from record messages """ If implementing in tap-tester framework the primary key implementation should account for compound primary keys """ - self.assertEqual(1, len(list(primary_keys[stream])), - msg="Compound primary keys require a change to test expectations") + self.assertEqual( + 1, + len(list(primary_keys[stream])), + msg="Compound primary keys require a change to test expectations", + ) primary_key = list(primary_keys[stream])[0] - incrmntl_sync_messages = [record['data'] for record in - incrmntl_sync_records.get(stream, {}).get('messages') - if record.get('action') == 'upsert'] - incrmntl_primary_keys = {message[primary_key] for message in incrmntl_sync_messages} - fulltbl_sync_messages = [record['data'] for record in - fulltbl_sync_records.get(stream, {}).get('messages') - if record.get('action') == 'upsert'] - filtered_fulltbl_sync_messages = [message for message in fulltbl_sync_messages - if message[replication_key] >= self.start_date] - fulltbl_primary_keys = {message[primary_key] for message in filtered_fulltbl_sync_messages} - - #Verify all records are synced in the second sync + incrmntl_sync_messages = [ + record["data"] + for record in incrmntl_sync_records.get(stream, {}).get("messages") + if record.get("action") == "upsert" + ] + incrmntl_primary_keys = { + message[primary_key] for message in incrmntl_sync_messages + } + fulltbl_sync_messages = [ + record["data"] + for record in fulltbl_sync_records.get(stream, {}).get("messages") + if record.get("action") == "upsert" + ] + filtered_fulltbl_sync_messages = [ + message + for message in fulltbl_sync_messages + if message[replication_key] >= self.start_date + ] + fulltbl_primary_keys = { + message[primary_key] for message in filtered_fulltbl_sync_messages + } + + # Verify all records are synced in the second sync self.assertTrue(incrmntl_primary_keys.issubset(fulltbl_primary_keys)) - #Verify that the fulltable sync count is greater or equal to incrmental sync count - self.assertGreaterEqual(fulltbl_sync_count, incrmntl_sync_count, - msg = "Full table sync didn't fetch all the records") + # Verify that the fulltable sync count is greater or equal to incrmental sync count + self.assertGreaterEqual( + fulltbl_sync_count, + incrmntl_sync_count, + msg="Full table sync didn't fetch all the records", + ) """ Modify the the activate version assertion accordingly based on the outcome of BUG #TDL-24467 if needed """ - #verify that last messages of every stream is the activate version message - self.assertEqual('activate_version', fulltbl_sync_records[stream]['messages'][-1] - ['action']) + # verify that last messages of every stream is the activate version message + self.assertEqual( + "activate_version", + fulltbl_sync_records[stream]["messages"][-1]["action"], + ) - #verify that table version is present for a fulltable sync - self.assertIsNotNone(fulltbl_sync_records[stream]['table_version']) + # verify that table version is present for a fulltable sync + self.assertIsNotNone(fulltbl_sync_records[stream]["table_version"]) - #Verify that the table version is incremented after every sync - self.assertGreater(fulltbl_sync_records[stream]['table_version'], - incrmntl_sync_records[stream]['table_version']) + # Verify that the table version is incremented after every sync + self.assertGreater( + fulltbl_sync_records[stream]["table_version"], + incrmntl_sync_records[stream]["table_version"], + ) # bookmarked states (top level objects) - fulltbl_bookmark_key_value = fulltbl_sync_bookmarks.get('bookmarks', {}).get(stream) + fulltbl_bookmark_key_value = fulltbl_sync_bookmarks.get( + "bookmarks", {} + ).get(stream) # bookmarked states (actual values) fulltbl_bookmark_value = fulltbl_bookmark_key_value.get(replication_key) - #verify no bookmarks are present in fulltbl sync + # verify no bookmarks are present in fulltbl sync self.assertIsNone(fulltbl_bookmark_value) diff --git a/tests/test_salesforce_sync_canary.py b/tests/test_salesforce_sync_canary.py index ea9936b6..a1dfa948 100644 --- a/tests/test_salesforce_sync_canary.py +++ b/tests/test_salesforce_sync_canary.py @@ -1,9 +1,5 @@ - -from datetime import datetime, timedelta - -from tap_tester import menagerie, connections, LOGGER - from sfbase import SFBaseTest +from tap_tester import connections, menagerie class SalesforceSyncCanary(SFBaseTest): @@ -19,24 +15,25 @@ def name(): @staticmethod def get_properties(): # pylint: disable=arguments-differ return { - 'start_date' : '2024-03-12T00:00:00Z', - 'instance_url': 'https://singer2-dev-ed.my.salesforce.com', - 'select_fields_by_default': 'true', - 'api_type': 'BULK', - 'is_sandbox': 'false' + "start_date": "2024-03-12T00:00:00Z", + "instance_url": "https://singer2-dev-ed.my.salesforce.com", + "select_fields_by_default": "true", + "api_type": "BULK", + "is_sandbox": "false", } def expected_sync_streams(self): - return self.expected_stream_names().difference({ - # DATACLOUD_API_DISABLED_EXCEPTION - 'DatacloudAddress', - 'DatacloudCompany', - 'DatacloudContact', - 'DatacloudDandBCompany', - 'DatacloudOwnedEntity', - 'DatacloudPurchaseUsage', - }) - + return self.expected_stream_names().difference( + { + # DATACLOUD_API_DISABLED_EXCEPTION + "DatacloudAddress", + "DatacloudCompany", + "DatacloudContact", + "DatacloudDandBCompany", + "DatacloudOwnedEntity", + "DatacloudPurchaseUsage", + } + ) def test_run(self): conn_id = connections.ensure_connection(self) @@ -46,14 +43,22 @@ def test_run(self): # select certain... catalogs expected_streams = self.partition_streams(self.expected_sync_streams()) - allowed_catalogs = [catalog for catalog in found_catalogs - if catalog['stream_name'] in expected_streams] + allowed_catalogs = [ + catalog + for catalog in found_catalogs + if catalog["stream_name"] in expected_streams + ] self.select_all_streams_and_fields(conn_id, allowed_catalogs) # Run sync menagerie.set_state(conn_id, {}) record_count_by_stream = self.run_and_verify_sync_mode(conn_id) - actual_streams_with_data ={stream for stream in record_count_by_stream - if record_count_by_stream[stream] > 0} - self.assertTrue(actual_streams_with_data.issubset(self.get_streams_with_data()), - msg = f"New streams with data are synced {actual_streams_with_data.difference(self.get_streams_with_data())}") + actual_streams_with_data = { + stream + for stream in record_count_by_stream + if record_count_by_stream[stream] > 0 + } + self.assertTrue( + actual_streams_with_data.issubset(self.get_streams_with_data()), + msg=f"New streams with data are synced {actual_streams_with_data.difference(self.get_streams_with_data())}", + ) diff --git a/tests/unittests/test_lookback_window.py b/tests/unittests/test_lookback_window.py index 2dd55850..d06eb76d 100644 --- a/tests/unittests/test_lookback_window.py +++ b/tests/unittests/test_lookback_window.py @@ -1,6 +1,7 @@ -from tap_salesforce.salesforce import Salesforce import unittest +from tap_salesforce.salesforce import Salesforce + start_date = "2022-05-02T00:00:00.000000Z" bookmark = "2022-05-23T00:00:00.000000Z" @@ -11,19 +12,13 @@ "breadcrumb": [], "metadata": { "replication-method": "INCREMENTAL", - "replication-key": "SystemModstamp" - } + "replication-key": "SystemModstamp", + }, } - ] + ], } -TEST_STATE = { - "bookmarks": { - "Test": { - "SystemModstamp": bookmark - } - } -} +TEST_STATE = {"bookmarks": {"Test": {"SystemModstamp": bookmark}}} TEST_LOOKBACK_WINDOW = 60 @@ -42,29 +37,20 @@ class SalesforceGetStartDateTests(unittest.TestCase): | Yes | No | start date | | Yes | Yes | adjusted bookmark | """ + def test_no_lookback_no_bookmark_returns_start_date(self): - sf_obj = Salesforce( - default_start_date=start_date - ) + sf_obj = Salesforce(default_start_date=start_date) expected = start_date - actual = sf_obj.get_start_date( - {}, - catalog_entry - ) + actual = sf_obj.get_start_date({}, catalog_entry) self.assertEqual(expected, actual) def test_no_lookback_yes_bookmark_returns_bookmark(self): - sf_obj = Salesforce( - default_start_date=start_date - ) + sf_obj = Salesforce(default_start_date=start_date) expected = bookmark - actual = sf_obj.get_start_date( - TEST_STATE, - catalog_entry - ) + actual = sf_obj.get_start_date(TEST_STATE, catalog_entry) self.assertEqual(expected, actual) @@ -75,23 +61,16 @@ def test_yes_lookback_no_bookmark_returns_start_date(self): ) expected = start_date - actual = sf_obj.get_start_date( - {}, - catalog_entry - ) + actual = sf_obj.get_start_date({}, catalog_entry) self.assertEqual(expected, actual) def test_yes_lookback_yes_bookmark_returns_adjusted_bookmark(self): sf_obj = Salesforce( - default_start_date=start_date, - lookback_window=TEST_LOOKBACK_WINDOW + default_start_date=start_date, lookback_window=TEST_LOOKBACK_WINDOW ) expected = "2022-05-22T23:59:00.000000Z" - actual = sf_obj.get_start_date( - TEST_STATE, - catalog_entry - ) + actual = sf_obj.get_start_date(TEST_STATE, catalog_entry) self.assertEqual(expected, actual) diff --git a/tests/unittests/test_multiline_critical_error_message.py b/tests/unittests/test_multiline_critical_error_message.py index 46759068..443f262c 100644 --- a/tests/unittests/test_multiline_critical_error_message.py +++ b/tests/unittests/test_multiline_critical_error_message.py @@ -1,7 +1,9 @@ import unittest from unittest import mock + from tap_salesforce import main + # mock "main_impl" and raise multiline error def raise_error(): raise Exception("""Error syncing Transaction__c: 400 Client Error: Bad Request for url: https://test.my.salesforce.com/services/async/41.0/job/7502K00000IcACtQAN/batch/test123j/result/test123b Response: Invalid session id """) + class TestMultiLineCriticalErrorMessage(unittest.TestCase): """ - Test case to verify every line in the multiline error contains 'CRITICAL' + Test case to verify every line in the multiline error contains 'CRITICAL' """ @mock.patch("tap_salesforce.LOGGER.critical") @mock.patch("tap_salesforce.main_impl") - def test_multiline_critical_error_message(self, mocked_main_impl, mocked_logger_critical): + def test_multiline_critical_error_message( + self, mocked_main_impl, mocked_logger_critical + ): # mock "main_impl" and raise multiline error mocked_main_impl.side_effect = raise_error @@ -29,9 +34,8 @@ def test_multiline_critical_error_message(self, mocked_main_impl, mocked_logger_ self.assertEqual(mocked_logger_critical.call_count, 5) @mock.patch("tap_salesforce.singer_utils.parse_args") - @mock.patch('tap_salesforce.salesforce.requests.Session.post') + @mock.patch("tap_salesforce.salesforce.requests.Session.post") def test_http_406_error_message(self, mocked_post, mocked_parse_args): - args = mock.MagicMock() args.config = { "refresh_token": "abc", @@ -42,7 +46,7 @@ def test_http_406_error_message(self, mocked_post, mocked_parse_args): "is_sandbox": True, "start_date": "2020-02-04T07:46:29Z", "api_type": "abc", - "lookback_window": "12" + "lookback_window": "12", } mocked_parse_args.return_value = args @@ -51,7 +55,6 @@ def test_http_406_error_message(self, mocked_post, mocked_parse_args): mock_response.status_code = 406 mocked_post.return_value = mock_response - # verify "Exception" is raise on function call with self.assertRaises(Exception): main() diff --git a/tests/unittests/test_salesforce_date_windowing.py b/tests/unittests/test_salesforce_date_windowing.py index 6f0e937b..7f1c557d 100644 --- a/tests/unittests/test_salesforce_date_windowing.py +++ b/tests/unittests/test_salesforce_date_windowing.py @@ -1,10 +1,13 @@ import datetime import unittest from unittest import mock + +import singer +from dateutil import tz + from tap_salesforce import Salesforce from tap_salesforce.salesforce import Bulk -from dateutil import tz -import singer + # function to return batch status as 'failed' to force date windowing def mocked_batch_status(count): @@ -13,72 +16,52 @@ def mocked_batch_status(count): else: return {"failed": {}} -@mock.patch('tap_salesforce.salesforce.Bulk._close_job') -@mock.patch('tap_salesforce.salesforce.Bulk._poll_on_pk_chunked_batch_status', side_effect = mocked_batch_status) -@mock.patch('tap_salesforce.salesforce.Bulk._create_job', side_effect=[1, 2, 3, 4, 5]) -@mock.patch('tap_salesforce.salesforce.bulk.singer_utils.now') -class TestBulkDateWindow(unittest.TestCase): +@mock.patch("tap_salesforce.salesforce.Bulk._close_job") +@mock.patch( + "tap_salesforce.salesforce.Bulk._poll_on_pk_chunked_batch_status", + side_effect=mocked_batch_status, +) +@mock.patch("tap_salesforce.salesforce.Bulk._create_job", side_effect=[1, 2, 3, 4, 5]) +@mock.patch("tap_salesforce.salesforce.bulk.singer_utils.now") +class TestBulkDateWindow(unittest.TestCase): # start date - start_date = '2019-02-04T12:15:00Z' + start_date = "2019-02-04T12:15:00Z" # 'Salesforce' object - sf = Salesforce( - default_start_date=start_date, - api_type="BULK") + sf = Salesforce(default_start_date=start_date, api_type="BULK") # dummy catalog entry catalog_entry = { - 'stream': 'User', - 'tap_stream_id': 'User', - 'schema': { + "stream": "User", + "tap_stream_id": "User", + "schema": { "properties": { - "Id": { - "type": "string" - }, + "Id": {"type": "string"}, "SystemModstamp": { - "anyOf": [{ - "type": "string", - "format": "date-time" - }, - { - "type": [ - "string", - "null" - ] - } + "anyOf": [ + {"type": "string", "format": "date-time"}, + {"type": ["string", "null"]}, ] - } + }, } }, - 'metadata': [ + "metadata": [ { "breadcrumb": [], "metadata": { "selected": True, "replication-key": "SystemModstamp", - "table-key-properties": [ - "Id" - ] - } + "table-key-properties": ["Id"], + }, }, { - "breadcrumb": [ - "properties", - "SystemModstamp" - ], - "metadata": { - "inclusion": "automatic" - } + "breadcrumb": ["properties", "SystemModstamp"], + "metadata": {"inclusion": "automatic"}, }, { - "breadcrumb": [ - "properties", - "Id" - ], - "metadata": { - "inclusion": "automatic" - } - } - ] + "breadcrumb": ["properties", "Id"], + "metadata": {"inclusion": "automatic"}, + }, + ], } # mocked now date @@ -88,24 +71,44 @@ class TestBulkDateWindow(unittest.TestCase): # mocked now date now_date_2 = datetime.datetime(2019, 2, 5, 12, 15, 00, tzinfo=tz.UTC) - @mock.patch('tap_salesforce.salesforce.Bulk._add_batch') - def test_bulk_date_windowing_with_max_retries_0(self, mocked_add_batch, mocked_singer_util_now, mocked_create_job, mocked_batch_status, mocked_close_job): + @mock.patch("tap_salesforce.salesforce.Bulk._add_batch") + def test_bulk_date_windowing_with_max_retries_0( + self, + mocked_add_batch, + mocked_singer_util_now, + mocked_create_job, + mocked_batch_status, + mocked_close_job, + ): """ - To verify that if data is too large then date windowing mechanism execute, + To verify that if data is too large then date windowing mechanism execute, but after retrying upto MAX_RETRIES still not get data then raise proper exception """ mocked_singer_util_now.return_value = self.now_date_1 with self.assertRaises(Exception) as e: - Bulk(self.sf)._bulk_with_window([], self.catalog_entry, self.start_date, retries=0) - - self.assertEqual(str(e.exception), 'Ran out of retries attempting to query Salesforce Object User', "Not get expected Exception") - - @mock.patch('tap_salesforce.salesforce.Bulk._add_batch') - def test_bulk_date_windowing_with_half_day_range_0(self, mocked_add_batch, mocked_singer_util_now, mocked_create_job, mocked_batch_status, mocked_close_job): + Bulk(self.sf)._bulk_with_window( + [], self.catalog_entry, self.start_date, retries=0 + ) + + self.assertEqual( + str(e.exception), + "Ran out of retries attempting to query Salesforce Object User", + "Not get expected Exception", + ) + + @mock.patch("tap_salesforce.salesforce.Bulk._add_batch") + def test_bulk_date_windowing_with_half_day_range_0( + self, + mocked_add_batch, + mocked_singer_util_now, + mocked_create_job, + mocked_batch_status, + mocked_close_job, + ): """ - To verify that if data is too large then date windowing mechanism execute, + To verify that if data is too large then date windowing mechanism execute, but after retrying window goes to 0 days, still not get data then raise proper exception """ @@ -114,11 +117,23 @@ def test_bulk_date_windowing_with_half_day_range_0(self, mocked_add_batch, mocke with self.assertRaises(Exception) as e: Bulk(self.sf)._bulk_with_window([], self.catalog_entry, self.start_date) - self.assertEqual(str(e.exception), 'Attempting to query by 0 day range, this would cause infinite looping.', "Not get expected Exception") - - @mock.patch('xmltodict.parse') - @mock.patch('tap_salesforce.salesforce.Salesforce._make_request') - def test_bulk_date_window_gaps(self, mocked_make_request, mocked_xmltodict_parse, mocked_singer_util_now, mocked_create_job, mocked_batch_status, mocked_close_job): + self.assertEqual( + str(e.exception), + "Attempting to query by 0 day range, this would cause infinite looping.", + "Not get expected Exception", + ) + + @mock.patch("xmltodict.parse") + @mock.patch("tap_salesforce.salesforce.Salesforce._make_request") + def test_bulk_date_window_gaps( + self, + mocked_make_request, + mocked_xmltodict_parse, + mocked_singer_util_now, + mocked_create_job, + mocked_batch_status, + mocked_close_job, + ): """ Test case to verify there are no gaps in the date window """ @@ -126,33 +141,35 @@ def test_bulk_date_window_gaps(self, mocked_make_request, mocked_xmltodict_parse # mock singer.now mocked_singer_util_now.return_value = self.now_date_1 # mock xmltodict.parse - mocked_xmltodict_parse.return_value = { - "batchInfo": { - "id": 1234 - } - } + mocked_xmltodict_parse.return_value = {"batchInfo": {"id": 1234}} # function call with start date Bulk(self.sf)._bulk_with_window([], self.catalog_entry, self.start_date) # collect 'body' (query) from '_make_request' function arguments - actual_queries = [kwargs.get("body") for args, kwargs in mocked_make_request.call_args_list] + actual_queries = [ + kwargs.get("body") for args, kwargs in mocked_make_request.call_args_list + ] # calculate half window date for assertion - half_day = (self.now_date_1 - singer.utils.strptime_with_tz(self.start_date)) // 2 - half_window_date = (self.now_date_1 - half_day).strftime('%Y-%m-%dT%H:%M:%SZ') + half_day = ( + self.now_date_1 - singer.utils.strptime_with_tz(self.start_date) + ) // 2 + half_window_date = (self.now_date_1 - half_day).strftime("%Y-%m-%dT%H:%M:%SZ") # create expected queries expected_queries = [ # failed call of whole date window ie. start date to now date - f'SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {self.start_date} AND SystemModstamp < {self.now_date_1_str}', + f"SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {self.start_date} AND SystemModstamp < {self.now_date_1_str}", # date window divided into half, query from start date to half window - f'SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {self.start_date} AND SystemModstamp < {half_window_date}', + f"SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {self.start_date} AND SystemModstamp < {half_window_date}", # query from half window to now date - f'SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {half_window_date} AND SystemModstamp < {self.now_date_1_str}' + f"SELECT Id,SystemModstamp FROM User WHERE SystemModstamp >= {half_window_date} AND SystemModstamp < {self.now_date_1_str}", ] # verify we called '_make_request' 3 times - self.assertEqual(mocked_make_request.call_count, 3, "Function is not called expected times") + self.assertEqual( + mocked_make_request.call_count, 3, "Function is not called expected times" + ) # verify the queries are called as expected self.assertEqual(actual_queries, expected_queries) diff --git a/tests/unittests/test_salesforce_null_bookmark.py b/tests/unittests/test_salesforce_null_bookmark.py index a477933a..4dadcf7c 100644 --- a/tests/unittests/test_salesforce_null_bookmark.py +++ b/tests/unittests/test_salesforce_null_bookmark.py @@ -1,24 +1,38 @@ +import json import unittest from unittest import mock + from tap_salesforce import Salesforce, metrics from tap_salesforce.sync import sync_records -import json + class TestNullBookmarkTesting(unittest.TestCase): - @mock.patch('tap_salesforce.salesforce.Salesforce.query', side_effect=lambda test1, test2: []) + @mock.patch( + "tap_salesforce.salesforce.Salesforce.query", + side_effect=lambda test1, test2: [], + ) def test_not_null_bookmark_for_incremental_stream(self, mocked_query): """ - To ensure that after resolving write bookmark logic not get "Null" as replication key in state file, + To ensure that after resolving write bookmark logic not get "Null" as replication key in state file, When we have selected incremental stream as Full table stream """ - sf = Salesforce(default_start_date='2019-02-04T12:15:00Z', api_type="BULK") + sf = Salesforce(default_start_date="2019-02-04T12:15:00Z", api_type="BULK") sf.pk_chunking = True - catalog_entry = {"stream": "OpportunityLineItem", "schema": {}, "metadata":[], "tap_stream_id": "OpportunityLineItem"} + catalog_entry = { + "stream": "OpportunityLineItem", + "schema": {}, + "metadata": [], + "tap_stream_id": "OpportunityLineItem", + } state = {} - counter = metrics.record_counter('OpportunityLineItem') + counter = metrics.record_counter("OpportunityLineItem") sync_records(sf, catalog_entry, state, counter) # write state function convert python dictionary to json string state = json.dumps(state) - self.assertEqual(state, '{"bookmarks": {"OpportunityLineItem": {"version": null}}}', "Not get expected state value") \ No newline at end of file + self.assertEqual( + state, + '{"bookmarks": {"OpportunityLineItem": {"version": null}}}', + "Not get expected state value", + )