diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca0fdf5e2d65..5d05d2b55d13 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.2.0 + rev: v0.3.0 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/Makefile b/Makefile index 928b85572d26..a0c945d38f30 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ unannotated_pyright: python scripts/run-pyright.py --unannotated ruff: - -ruff --fix . + -ruff check --fix . ruff format . check_ruff: diff --git a/docs/Makefile b/docs/Makefile index a51494d665fb..97515c2fc285 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,5 +1,5 @@ docs_ruff: - -ruff --fix ../examples/docs_snippets + -ruff check --fix ../examples/docs_snippets ruff format ../examples/docs_snippets apidoc-build: diff --git a/docs/content/concepts/assets/asset-auto-execution.mdx b/docs/content/concepts/assets/asset-auto-execution.mdx index fe6e8e673ab3..f6f3e3bc98f2 100644 --- a/docs/content/concepts/assets/asset-auto-execution.mdx +++ b/docs/content/concepts/assets/asset-auto-execution.mdx @@ -39,13 +39,11 @@ from dagster import AutoMaterializePolicy, asset @asset -def asset1(): - ... +def asset1(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.eager(), deps=[asset1]) -def asset2(): - ... +def asset2(): ... ``` This example assumes that `asset1` will be materialized in some other way - e.g. manually, via a [sensor](/concepts/partitions-schedules-sensors/sensors), or via a [schedule](/concepts/partitions-schedules-sensors/schedules). @@ -64,13 +62,11 @@ from dagster import ( @asset -def asset1(): - ... +def asset1(): ... @asset(deps=[asset1]) -def asset2(): - ... +def asset2(): ... defs = Definitions( @@ -101,8 +97,7 @@ wait_for_all_parents_policy = AutoMaterializePolicy.eager().with_rules( @asset(auto_materialize_policy=wait_for_all_parents_policy) -def asset1(upstream1, upstream2): - ... +def asset1(upstream1, upstream2): ... ``` #### Auto-materialize even if some parents are missing @@ -118,8 +113,7 @@ allow_missing_parents_policy = AutoMaterializePolicy.eager().without_rules( @asset(auto_materialize_policy=allow_missing_parents_policy) -def asset1(upstream1, upstream2): - ... +def asset1(upstream1, upstream2): ... ``` #### Auto-materialize root assets on a regular cadence @@ -136,8 +130,7 @@ materialize_on_cron_policy = AutoMaterializePolicy.eager().with_rules( @asset(auto_materialize_policy=materialize_on_cron_policy) -def root_asset(): - ... +def root_asset(): ... ``` ### Auto-materialization and partitions @@ -152,8 +145,7 @@ from dagster import AutoMaterializePolicy, DailyPartitionsDefinition, asset partitions_def=DailyPartitionsDefinition(start_date="2020-10-10"), auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def asset1(): - ... +def asset1(): ... @asset( @@ -161,8 +153,7 @@ def asset1(): auto_materialize_policy=AutoMaterializePolicy.eager(), deps=[asset1], ) -def asset2(): - ... +def asset2(): ... ``` If the last partition of `asset1` is re-materialized, e.g. manually from the UI, then the corresponding partition of `asset2` will be auto-materialized after. @@ -181,8 +172,7 @@ from dagster import AutoMaterializePolicy, DailyPartitionsDefinition, asset max_materializations_per_minute=7 ), ) -def asset1(): - ... +def asset1(): ... ``` For time-partitioned assets, the `N` most recent partitions will be selected from the set of candidates to be materialized. For other types of partitioned assets, the selection will be random. @@ -208,6 +198,5 @@ def source_file(): deps=[source_file], auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def asset1(): - ... +def asset1(): ... ``` diff --git a/docs/content/concepts/assets/asset-checks.mdx b/docs/content/concepts/assets/asset-checks.mdx index c129ff37c5eb..af558b8ea77a 100644 --- a/docs/content/concepts/assets/asset-checks.mdx +++ b/docs/content/concepts/assets/asset-checks.mdx @@ -113,8 +113,7 @@ from dagster import ( @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset=my_asset) @@ -184,13 +183,11 @@ from dagster import ( @asset -def orders(): - ... +def orders(): ... @asset -def items(): - ... +def items(): ... def make_check(check_blob: Mapping[str, str]) -> AssetChecksDefinition: @@ -250,18 +247,15 @@ from dagster import ( @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset=my_asset) -def check_1(): - ... +def check_1(): ... @asset_check(asset=my_asset) -def check_2(): - ... +def check_2(): ... # includes my_asset and both checks diff --git a/docs/content/concepts/configuration/advanced-config-types.mdx b/docs/content/concepts/configuration/advanced-config-types.mdx index 65a247e14683..4e5ac2f94189 100644 --- a/docs/content/concepts/configuration/advanced-config-types.mdx +++ b/docs/content/concepts/configuration/advanced-config-types.mdx @@ -124,8 +124,7 @@ class MyDataStructuresConfig(Config): user_scores: Dict[str, int] @asset -def scoreboard(config: MyDataStructuresConfig): - ... +def scoreboard(config: MyDataStructuresConfig): ... result = materialize( [scoreboard], @@ -161,8 +160,7 @@ class MyNestedConfig(Config): user_data: Dict[str, UserData] @asset -def average_age(config: MyNestedConfig): - ... +def average_age(config: MyNestedConfig): ... result = materialize( [average_age], diff --git a/docs/content/concepts/io-management/io-managers.mdx b/docs/content/concepts/io-management/io-managers.mdx index 511123c8ace3..cff2e0f9e72a 100644 --- a/docs/content/concepts/io-management/io-managers.mdx +++ b/docs/content/concepts/io-management/io-managers.mdx @@ -405,8 +405,7 @@ class ExternalIOManager(IOManager): # setup stateful cache self._cache = {} - def handle_output(self, context: OutputContext, obj): - ... + def handle_output(self, context: OutputContext, obj): ... def load_input(self, context: InputContext): if context.asset_key in self._cache: diff --git a/docs/content/concepts/logging/loggers.mdx b/docs/content/concepts/logging/loggers.mdx index 6556dad6dc47..6debc1299f95 100644 --- a/docs/content/concepts/logging/loggers.mdx +++ b/docs/content/concepts/logging/loggers.mdx @@ -203,8 +203,7 @@ from dagster import Definitions, define_asset_job, asset @asset -def some_asset(): - ... +def some_asset(): ... the_job = define_asset_job("the_job", selection="*") diff --git a/docs/content/concepts/ops-jobs-graphs/graphs.mdx b/docs/content/concepts/ops-jobs-graphs/graphs.mdx index 8b38e2727445..669093c6766a 100644 --- a/docs/content/concepts/ops-jobs-graphs/graphs.mdx +++ b/docs/content/concepts/ops-jobs-graphs/graphs.mdx @@ -272,13 +272,11 @@ from dagster import asset, job, op @asset -def emails_to_send(): - ... +def emails_to_send(): ... @op -def send_emails(emails) -> None: - ... +def send_emails(emails) -> None: ... @job diff --git a/docs/content/concepts/ops-jobs-graphs/job-execution.mdx b/docs/content/concepts/ops-jobs-graphs/job-execution.mdx index de4811ba4d3d..d1155d16b3c2 100644 --- a/docs/content/concepts/ops-jobs-graphs/job-execution.mdx +++ b/docs/content/concepts/ops-jobs-graphs/job-execution.mdx @@ -220,8 +220,7 @@ For example, the following job will execute at most two ops at once with the `da } } ) -def tag_concurrency_job(): - ... +def tag_concurrency_job(): ... ``` **Note:** These limits are only applied on a per-run basis. You can apply op concurrency limits across multiple runs using the or . diff --git a/docs/content/concepts/ops-jobs-graphs/op-jobs.mdx b/docs/content/concepts/ops-jobs-graphs/op-jobs.mdx index 2ec9663bd54d..24004c1264f8 100644 --- a/docs/content/concepts/ops-jobs-graphs/op-jobs.mdx +++ b/docs/content/concepts/ops-jobs-graphs/op-jobs.mdx @@ -87,8 +87,7 @@ from dagster import graph, op, ConfigurableResource class Server(ConfigurableResource): - def ping_server(self): - ... + def ping_server(self): ... @op @@ -222,8 +221,7 @@ from dagster import Definitions, job @job -def do_it_all(): - ... +def do_it_all(): ... defs = Definitions(jobs=[do_it_all]) diff --git a/docs/content/concepts/partitions-schedules-sensors/partitioning-assets.mdx b/docs/content/concepts/partitions-schedules-sensors/partitioning-assets.mdx index 3dc06dce7f18..55fcdbda662d 100644 --- a/docs/content/concepts/partitions-schedules-sensors/partitioning-assets.mdx +++ b/docs/content/concepts/partitions-schedules-sensors/partitioning-assets.mdx @@ -156,8 +156,7 @@ images_partitions_def = DynamicPartitionsDefinition(name="images") @asset(partitions_def=images_partitions_def) -def images(context: AssetExecutionContext): - ... +def images(context: AssetExecutionContext): ... ``` Partition keys can be added and removed for a given dynamic partition set. For example, the following code snippet demonstrates the usage of a [sensor](/concepts/partitions-schedules-sensors/sensors) to detect the presence of a new partition and then trigger a run for that partition: @@ -256,8 +255,7 @@ partitions_def = DailyPartitionsDefinition(start_date="2023-01-21") @asset(partitions_def=partitions_def) -def events(): - ... +def events(): ... @asset( @@ -271,8 +269,7 @@ def events(): ) ], ) -def yesterday_event_stats(): - ... +def yesterday_event_stats(): ... ``` @@ -296,8 +293,7 @@ partitions_def = DailyPartitionsDefinition(start_date="2023-01-21") @asset(partitions_def=partitions_def) -def events(): - ... +def events(): ... @asset( @@ -310,8 +306,7 @@ def events(): ) }, ) -def yesterday_event_stats(events): - ... +def yesterday_event_stats(events): ... ``` @@ -340,13 +335,11 @@ hourly_partitions_def = HourlyPartitionsDefinition(start_date="2022-05-31-00:00" @asset(partitions_def=hourly_partitions_def) -def asset1(): - ... +def asset1(): ... @asset(partitions_def=hourly_partitions_def) -def asset2(): - ... +def asset2(): ... partitioned_asset_job = define_asset_job( diff --git a/docs/content/concepts/partitions-schedules-sensors/partitioning-ops.mdx b/docs/content/concepts/partitions-schedules-sensors/partitioning-ops.mdx index 80f6d930251a..aca9a6110a21 100644 --- a/docs/content/concepts/partitions-schedules-sensors/partitioning-ops.mdx +++ b/docs/content/concepts/partitions-schedules-sensors/partitioning-ops.mdx @@ -164,8 +164,7 @@ from dagster import build_schedule_from_partitioned_job, job @job(config=my_partitioned_config) -def do_stuff_partitioned(): - ... +def do_stuff_partitioned(): ... do_stuff_partitioned_schedule = build_schedule_from_partitioned_job( diff --git a/docs/content/concepts/partitions-schedules-sensors/schedules.mdx b/docs/content/concepts/partitions-schedules-sensors/schedules.mdx index 790b73955bb5..13d0780cb2f3 100644 --- a/docs/content/concepts/partitions-schedules-sensors/schedules.mdx +++ b/docs/content/concepts/partitions-schedules-sensors/schedules.mdx @@ -45,8 +45,7 @@ Here's a simple schedule that runs a job every day, at midnight: ```python file=concepts/partitions_schedules_sensors/schedules/schedules.py startafter=start_basic_schedule endbefore=end_basic_schedule @job -def my_job(): - ... +def my_job(): ... basic_schedule = ScheduleDefinition(job=my_job, cron_schedule="0 0 * * *") @@ -109,8 +108,7 @@ from dagster import build_schedule_from_partitioned_job, job @job(config=my_partitioned_config) -def do_stuff_partitioned(): - ... +def do_stuff_partitioned(): ... do_stuff_partitioned_schedule = build_schedule_from_partitioned_job( @@ -130,8 +128,7 @@ from dagster import ( @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) -def hourly_asset(): - ... +def hourly_asset(): ... partitioned_asset_job = define_asset_job("partitioned_job", selection=[hourly_asset]) @@ -264,8 +261,7 @@ class DateFormatter(ConfigurableResource): return dt.strftime(self.format) @job -def process_data(): - ... +def process_data(): ... @schedule(job=process_data, cron_schedule="* * * * *") def process_data_schedule( diff --git a/docs/content/concepts/partitions-schedules-sensors/sensors.mdx b/docs/content/concepts/partitions-schedules-sensors/sensors.mdx index 0eb86cd255d8..5c11fa40fdcf 100644 --- a/docs/content/concepts/partitions-schedules-sensors/sensors.mdx +++ b/docs/content/concepts/partitions-schedules-sensors/sensors.mdx @@ -97,8 +97,7 @@ Once a sensor is added to a object with the jo ```python file=concepts/partitions_schedules_sensors/sensors/sensors.py startafter=start_running_in_code endbefore=end_running_in_code @sensor(job=asset_job, default_status=DefaultSensorStatus.RUNNING) -def my_running_sensor(): - ... +def my_running_sensor(): ... ``` If you manually start or stop a sensor in the UI, that will override any default status that is set in code. @@ -250,8 +249,7 @@ class UsersAPI(ConfigurableResource): return requests.get(self.url).json() @job -def process_user(): - ... +def process_user(): ... @sensor(job=process_user) def process_new_users_sensor( diff --git a/docs/content/concepts/resources-legacy.mdx b/docs/content/concepts/resources-legacy.mdx index d9d1ccd7090e..90a8ec88a19c 100644 --- a/docs/content/concepts/resources-legacy.mdx +++ b/docs/content/concepts/resources-legacy.mdx @@ -305,8 +305,7 @@ from dagster import resource, build_resources @resource -def the_credentials(): - ... +def the_credentials(): ... @resource(required_resource_keys={"credentials"}) diff --git a/docs/content/concepts/resources.mdx b/docs/content/concepts/resources.mdx index 1ddafa8dc4ba..64f8a7d08a02 100644 --- a/docs/content/concepts/resources.mdx +++ b/docs/content/concepts/resources.mdx @@ -196,8 +196,7 @@ from dagster import ConfigurableResource, Definitions, asset class DatabaseResource(ConfigurableResource): table: str - def read(self): - ... + def read(self): ... @asset def data_from_database(db_conn: DatabaseResource): @@ -348,12 +347,10 @@ from pydantic import PrivateAttr class DBConnection: ... - def query(self, body: str): - ... + def query(self, body: str): ... @contextmanager -def get_database_connection(username: str, password: str): - ... +def get_database_connection(username: str, password: str): ... class MyClientResource(ConfigurableResource): username: str diff --git a/docs/content/concepts/testing.mdx b/docs/content/concepts/testing.mdx index 314dc41eca28..4effd0a01e29 100644 --- a/docs/content/concepts/testing.mdx +++ b/docs/content/concepts/testing.mdx @@ -94,8 +94,7 @@ from dagster import graph, op, ConfigurableResource, OpExecutionContext class MyApi(ConfigurableResource): - def call(self): - ... + def call(self): ... @op @@ -248,8 +247,7 @@ import mock class MyClient: ... - def query(self, body: str): - ... + def query(self, body: str): ... class MyClientResource(ConfigurableResource): username: str @@ -412,18 +410,15 @@ from dagster import asset, materialize_to_memory, ConfigurableResource import mock -class MyServiceResource(ConfigurableResource): - ... +class MyServiceResource(ConfigurableResource): ... @asset -def asset_requires_service(service: MyServiceResource): - ... +def asset_requires_service(service: MyServiceResource): ... @asset -def other_asset_requires_service(service: MyServiceResource): - ... +def other_asset_requires_service(service: MyServiceResource): ... def test_assets_require_service(): diff --git a/docs/content/deployment/executors.mdx b/docs/content/deployment/executors.mdx index 1365ab45db77..14e175dbacd1 100644 --- a/docs/content/deployment/executors.mdx +++ b/docs/content/deployment/executors.mdx @@ -37,13 +37,11 @@ from dagster import graph, job, multiprocess_executor # Providing an executor using the job decorator @job(executor_def=multiprocess_executor) -def the_job(): - ... +def the_job(): ... @graph -def the_graph(): - ... +def the_graph(): ... # Providing an executor using graph_def.to_job(...) @@ -69,8 +67,7 @@ asset_job = define_asset_job("the_job", selection="*") @job -def op_job(): - ... +def op_job(): ... # op_job and asset_job will both use the multiprocess_executor, diff --git a/docs/content/deployment/guides/gcp.mdx b/docs/content/deployment/guides/gcp.mdx index 10224fdd4aec..913cf836d6e3 100644 --- a/docs/content/deployment/guides/gcp.mdx +++ b/docs/content/deployment/guides/gcp.mdx @@ -61,8 +61,7 @@ from dagster import job } }, ) -def gcs_job(): - ... +def gcs_job(): ... ``` With this in place, your job runs will store outputs on GCS in the location `gs:///dagster/storage//files/.compute`. diff --git a/docs/content/deployment/run-monitoring.mdx b/docs/content/deployment/run-monitoring.mdx index b2db2295efff..141c42f5e6c7 100644 --- a/docs/content/deployment/run-monitoring.mdx +++ b/docs/content/deployment/run-monitoring.mdx @@ -40,8 +40,7 @@ from dagster import define_asset_job, job @job(tags={"dagster/max_runtime": 10}) -def my_job(): - ... +def my_job(): ... asset_job = define_asset_job( diff --git a/docs/content/guides/customizing-run-queue-priority.mdx b/docs/content/guides/customizing-run-queue-priority.mdx index 9fa8e8f15afe..c0a9fd85416c 100644 --- a/docs/content/guides/customizing-run-queue-priority.mdx +++ b/docs/content/guides/customizing-run-queue-priority.mdx @@ -51,8 +51,7 @@ In this example, the priority is set to `-1` with a `dagster/priority` tag value ```python startafter=start_marker_priority endbefore=end_marker_priority file=/deploying/concurrency_limits/concurrency_limits.py @job(tags={"dagster/priority": "3"}) -def important_job(): - ... +def important_job(): ... @schedule( @@ -61,8 +60,7 @@ def important_job(): execution_timezone="US/Central", tags={"dagster/priority": "-1"}, ) -def less_important_schedule(_): - ... +def less_important_schedule(_): ... ``` diff --git a/docs/content/guides/dagster-pipes/subprocess/reference.mdx b/docs/content/guides/dagster-pipes/subprocess/reference.mdx index 470c6b95bd29..c23b3e93f1db 100644 --- a/docs/content/guides/dagster-pipes/subprocess/reference.mdx +++ b/docs/content/guides/dagster-pipes/subprocess/reference.mdx @@ -144,8 +144,7 @@ from dagster import ( @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset="my_asset") diff --git a/docs/content/guides/dagster/how-assets-relate-to-ops-and-graphs.mdx b/docs/content/guides/dagster/how-assets-relate-to-ops-and-graphs.mdx index 33d6e803028f..a0bd0baeaae4 100644 --- a/docs/content/guides/dagster/how-assets-relate-to-ops-and-graphs.mdx +++ b/docs/content/guides/dagster/how-assets-relate-to-ops-and-graphs.mdx @@ -138,13 +138,11 @@ from dagster import asset, job, op @asset -def emails_to_send(): - ... +def emails_to_send(): ... @op -def send_emails(emails) -> None: - ... +def send_emails(emails) -> None: ... @job diff --git a/docs/content/guides/dagster/managing-ml.mdx b/docs/content/guides/dagster/managing-ml.mdx index a55d80c80640..f966c5063edd 100644 --- a/docs/content/guides/dagster/managing-ml.mdx +++ b/docs/content/guides/dagster/managing-ml.mdx @@ -35,15 +35,13 @@ from dagster import AutoMaterializePolicy, asset @asset -def my_data(): - ... +def my_data(): ... @asset( auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def my_ml_model(my_data): - ... +def my_ml_model(my_data): ... ``` Some machine learning models might more be cumbersome to retrain; it also might be less important to update them as soon as new data arrives. For this, a lazy auto-materialization policy which can be used in two different ways. The first, by using it with a `freshness_policy` as shown below. In this case, `my_ml_model` will only be auto-materialized once a week. @@ -53,16 +51,14 @@ from dagster import AutoMaterializePolicy, asset, FreshnessPolicy @asset -def my_other_data(): - ... +def my_other_data(): ... @asset( auto_materialize_policy=AutoMaterializePolicy.lazy(), freshness_policy=FreshnessPolicy(maximum_lag_minutes=7 * 24 * 60), ) -def my_other_ml_model(my_other_data): - ... +def my_other_ml_model(my_other_data): ... ``` This can be useful if you know that you want your machine learning model retrained at least once a week. While Dagster allows you to refresh a machine learning model as often as you like, best practice is to re-train as seldomly as possible. Model retraining can be costly to compute and having a minimal number of model versions can reduce the complexity of reproducing results at a later point in time. In this case, the model is updated once a week for `predictions`, ensuring that `my_ml_model` is retrained before it is used. @@ -72,21 +68,18 @@ from dagster import AutoMaterializePolicy, FreshnessPolicy, asset @asset -def some_data(): - ... +def some_data(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.lazy()) -def some_ml_model(some_data): - ... +def some_ml_model(some_data): ... @asset( auto_materialize_policy=AutoMaterializePolicy.lazy(), freshness_policy=FreshnessPolicy(maximum_lag_minutes=7 * 24 * 60), ) -def predictions(some_ml_model): - ... +def predictions(some_ml_model): ... ``` A more traditional schedule can also be used to update machine learning assets, causing them to be re-materialized or retrained on the latest data. For example, setting up a [cron schedule on a daily basis](/concepts/partitions-schedules-sensors/schedules). diff --git a/docs/content/guides/dagster/migrating-to-pythonic-resources-and-config.mdx b/docs/content/guides/dagster/migrating-to-pythonic-resources-and-config.mdx index 9c5a945afc9f..7502a518402e 100644 --- a/docs/content/guides/dagster/migrating-to-pythonic-resources-and-config.mdx +++ b/docs/content/guides/dagster/migrating-to-pythonic-resources-and-config.mdx @@ -91,8 +91,7 @@ class FancyDbResource: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... @resource(config_schema={"conn_string": str}) def fancy_db_resource(context: InitResourceContext) -> FancyDbResource: @@ -124,8 +123,7 @@ from dagster import ConfigurableResource class FancyDbResource(ConfigurableResource): conn_string: str - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... ``` The attributes declared on a class inheriting from serve as the new way to declare a configuration schema. Now, however, there's a problem: You're migrating an existing codebase that contains numerous callsites to the old `fancy_db_resource` function annotated with `@resource`. You have declared the config schema twice, once on `@resource` and once on the class. This is fine for now as the config schema is simple, but for more complicated schemas this can be a problem. @@ -161,8 +159,7 @@ from dagster import AssetExecutionContext, ConfigurableResource, Definitions, as class FancyDbResource(ConfigurableResource): conn_string: str - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... @asset(required_resource_keys={"fancy_db"}) def asset_one(context: AssetExecutionContext) -> None: @@ -204,8 +201,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # Alternatively could have been imported from third-party library # from fancy_db import FancyDbClient diff --git a/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx b/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx index 1dd600b24e48..64f489454129 100644 --- a/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx +++ b/docs/content/guides/limiting-concurrency-in-data-pipelines.mdx @@ -277,8 +277,7 @@ Op-based jobs are defined using the decorato } } ) -def tag_concurrency_job(): - ... +def tag_concurrency_job(): ... ``` @@ -370,8 +369,7 @@ In this example, using the Optional[Dict[str, Any]]: - ... + def fetch_item_by_id(self, item_id: int) -> Optional[Dict[str, Any]]: ... @abstractmethod - def fetch_max_item_id(self) -> int: - ... + def fetch_max_item_id(self) -> int: ... @property @abstractmethod - def item_field_names(self) -> Sequence[str]: - ... + def item_field_names(self) -> Sequence[str]: ... class HNAPIClient(HNClient): diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/factory.py b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/factory.py index d993530e63bd..c29f91e18fd7 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/factory.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/factory.py @@ -12,13 +12,11 @@ @asset -def orders(): - ... +def orders(): ... @asset -def items(): - ... +def items(): ... def make_check(check_blob: Mapping[str, str]) -> AssetChecksDefinition: diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/jobs.py b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/jobs.py index 25f68fd5e37d..e8ed84d175c0 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/jobs.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/jobs.py @@ -9,18 +9,15 @@ @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset=my_asset) -def check_1(): - ... +def check_1(): ... @asset_check(asset=my_asset) -def check_2(): - ... +def check_2(): ... # includes my_asset and both checks diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/severity.py b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/severity.py index d4a215d40a65..48215d01e6cc 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/severity.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/asset_checks/severity.py @@ -8,8 +8,7 @@ @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset=my_asset) diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_after_all_parents.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_after_all_parents.py index e0417d080344..bc2361ea1ba5 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_after_all_parents.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_after_all_parents.py @@ -6,5 +6,4 @@ @asset(auto_materialize_policy=wait_for_all_parents_policy) -def asset1(upstream1, upstream2): - ... +def asset1(upstream1, upstream2): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_eager.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_eager.py index 0183b2fc17f7..b1add0fadf22 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_eager.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_eager.py @@ -2,10 +2,8 @@ @asset -def asset1(): - ... +def asset1(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.eager(), deps=[asset1]) -def asset2(): - ... +def asset2(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy.py index 1ab3c4a7f9c8..f59128710649 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy.py @@ -2,8 +2,7 @@ @asset -def asset1(): - ... +def asset1(): ... @asset( @@ -11,5 +10,4 @@ def asset1(): freshness_policy=FreshnessPolicy(maximum_lag_minutes=24 * 60), deps=[asset1], ) -def asset2(): - ... +def asset2(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy_transitive.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy_transitive.py index c775a57a5d1b..a26fba228f3c 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy_transitive.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_lazy_transitive.py @@ -2,13 +2,11 @@ @asset -def asset1(): - ... +def asset1(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.lazy(), deps=[asset1]) -def asset2(): - ... +def asset2(): ... @asset( @@ -16,5 +14,4 @@ def asset2(): freshness_policy=FreshnessPolicy(maximum_lag_minutes=24 * 60), deps=[asset2], ) -def asset3(): - ... +def asset3(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_max_materializations_per_minute.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_max_materializations_per_minute.py index c463986a152f..8182eb0f8ba1 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_max_materializations_per_minute.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_max_materializations_per_minute.py @@ -7,5 +7,4 @@ max_materializations_per_minute=7 ), ) -def asset1(): - ... +def asset1(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_multiple.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_multiple.py index cd8d85cb2d1e..adbe982a7f24 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_multiple.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_multiple.py @@ -7,13 +7,11 @@ @asset -def asset1(): - ... +def asset1(): ... @asset(deps=[asset1]) -def asset2(): - ... +def asset2(): ... defs = Definitions( diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_observable_source_asset.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_observable_source_asset.py index 28e342a556df..0cfd6f1e5765 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_observable_source_asset.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_observable_source_asset.py @@ -12,5 +12,4 @@ def source_file(): deps=[source_file], auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def asset1(): - ... +def asset1(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_on_cron.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_on_cron.py index 0c775c03d5a1..2a83f793484a 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_on_cron.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_on_cron.py @@ -7,5 +7,4 @@ @asset(auto_materialize_policy=materialize_on_cron_policy) -def root_asset(): - ... +def root_asset(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_time_partitions.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_time_partitions.py index a1b6e621e495..837691d9fee1 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_time_partitions.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_time_partitions.py @@ -5,8 +5,7 @@ partitions_def=DailyPartitionsDefinition(start_date="2020-10-10"), auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def asset1(): - ... +def asset1(): ... @asset( @@ -14,5 +13,4 @@ def asset1(): auto_materialize_policy=AutoMaterializePolicy.eager(), deps=[asset1], ) -def asset2(): - ... +def asset2(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_with_missing_parents.py b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_with_missing_parents.py index a09bb837d8ea..ca00b76e006a 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_with_missing_parents.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/auto_materialize_with_missing_parents.py @@ -6,5 +6,4 @@ @asset(auto_materialize_policy=allow_missing_parents_policy) -def asset1(upstream1, upstream2): - ... +def asset1(upstream1, upstream2): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/assets/load_asset_values.py b/examples/docs_snippets/docs_snippets/concepts/assets/load_asset_values.py index a8c8fb030126..63892ddddc4c 100644 --- a/examples/docs_snippets/docs_snippets/concepts/assets/load_asset_values.py +++ b/examples/docs_snippets/docs_snippets/concepts/assets/load_asset_values.py @@ -20,12 +20,10 @@ def load_input(self, context: InputContext): def get_assets(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... return with_resources( [asset1, asset2], diff --git a/examples/docs_snippets/docs_snippets/concepts/io_management/custom_io_manager.py b/examples/docs_snippets/docs_snippets/concepts/io_management/custom_io_manager.py index 93e3bb7e4e06..beec13b89220 100755 --- a/examples/docs_snippets/docs_snippets/concepts/io_management/custom_io_manager.py +++ b/examples/docs_snippets/docs_snippets/concepts/io_management/custom_io_manager.py @@ -62,8 +62,7 @@ def __init__(self, api_token): # setup stateful cache self._cache = {} - def handle_output(self, context: OutputContext, obj): - ... + def handle_output(self, context: OutputContext, obj): ... def load_input(self, context: InputContext): if context.asset_key in self._cache: diff --git a/examples/docs_snippets/docs_snippets/concepts/logging/custom_logger.py b/examples/docs_snippets/docs_snippets/concepts/logging/custom_logger.py index 65fe5cfccadb..4ccbb858a44d 100644 --- a/examples/docs_snippets/docs_snippets/concepts/logging/custom_logger.py +++ b/examples/docs_snippets/docs_snippets/concepts/logging/custom_logger.py @@ -75,8 +75,7 @@ def test_init_json_console_logger_with_context(): @asset -def some_asset(): - ... +def some_asset(): ... the_job = define_asset_job("the_job", selection="*") diff --git a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/job_execution.py b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/job_execution.py index 357d407ff3a1..bb832f16ecc2 100644 --- a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/job_execution.py +++ b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/job_execution.py @@ -97,8 +97,7 @@ def forkserver_job(): } } ) -def tag_concurrency_job(): - ... +def tag_concurrency_job(): ... # end_tag_concurrency diff --git a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/jobs_from_graphs.py b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/jobs_from_graphs.py index f580163426db..71e305d0b64f 100644 --- a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/jobs_from_graphs.py +++ b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/jobs_from_graphs.py @@ -5,8 +5,7 @@ class Server(ConfigurableResource): - def ping_server(self): - ... + def ping_server(self): ... @op diff --git a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/repo_with_job.py b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/repo_with_job.py index b3fcf349db0c..6a50f158ae00 100644 --- a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/repo_with_job.py +++ b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/repo_with_job.py @@ -2,8 +2,7 @@ @job -def do_it_all(): - ... +def do_it_all(): ... defs = Definitions(jobs=[do_it_all]) diff --git a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/unit_tests.py b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/unit_tests.py index 4bf30b35fc95..5922999ca041 100644 --- a/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/unit_tests.py +++ b/examples/docs_snippets/docs_snippets/concepts/ops_jobs_graphs/unit_tests.py @@ -179,8 +179,7 @@ class Client: def __init__(self, api_key): self._api_key = api_key - def query(self, body): - ... + def query(self, body): ... class MyClientResource(ConfigurableResource): @@ -411,18 +410,15 @@ def test_data_assets(): import mock -class MyServiceResource(ConfigurableResource): - ... +class MyServiceResource(ConfigurableResource): ... @asset -def asset_requires_service(service: MyServiceResource): - ... +def asset_requires_service(service: MyServiceResource): ... @asset -def other_asset_requires_service(service: MyServiceResource): - ... +def other_asset_requires_service(service: MyServiceResource): ... def test_assets_require_service(): diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_asset.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_asset.py index dc7d11cfb2c1..46b783fd2882 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_asset.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_asset.py @@ -25,13 +25,10 @@ def events(context: AssetExecutionContext): # end_marker -def compute_events_from_raw_events(*args): - ... +def compute_events_from_raw_events(*args): ... -def read_data_in_datetime_range(*args): - ... +def read_data_in_datetime_range(*args): ... -def overwrite_data_in_datetime_range(*args): - ... +def overwrite_data_in_datetime_range(*args): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_io_manager.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_io_manager.py index ddddc431810c..7f9edc1595dd 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_io_manager.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/backfills/single_run_backfill_io_manager.py @@ -15,9 +15,7 @@ def handle_output(self, context: OutputContext, obj): # end_marker -def read_data_in_datetime_range(*args): - ... +def read_data_in_datetime_range(*args): ... -def overwrite_data_in_datetime_range(*args): - ... +def overwrite_data_in_datetime_range(*args): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/dynamic_partitioned_asset.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/dynamic_partitioned_asset.py index 672d27486678..fe9b5ea38772 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/dynamic_partitioned_asset.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/dynamic_partitioned_asset.py @@ -18,8 +18,7 @@ @asset(partitions_def=images_partitions_def) -def images(context: AssetExecutionContext): - ... +def images(context: AssetExecutionContext): ... # end_dynamic_partitions_marker diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partition_mapping.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partition_mapping.py index 72fd491d5166..21307a8d1d1e 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partition_mapping.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partition_mapping.py @@ -9,8 +9,7 @@ @asset(partitions_def=partitions_def) -def events(): - ... +def events(): ... @asset( @@ -23,5 +22,4 @@ def events(): ) }, ) -def yesterday_event_stats(events): - ... +def yesterday_event_stats(events): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_job.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_job.py index b66aa2372ca6..94e6f593afbd 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_job.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_job.py @@ -10,13 +10,11 @@ @asset(partitions_def=hourly_partitions_def) -def asset1(): - ... +def asset1(): ... @asset(partitions_def=hourly_partitions_def) -def asset2(): - ... +def asset2(): ... partitioned_asset_job = define_asset_job( diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_mappings.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_mappings.py index 3e337c83de13..5ead37ba5f8f 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_mappings.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/partitioned_asset_mappings.py @@ -9,8 +9,7 @@ @asset(partitions_def=partitions_def) -def events(): - ... +def events(): ... @asset( @@ -24,5 +23,4 @@ def events(): ) ], ) -def yesterday_event_stats(): - ... +def yesterday_event_stats(): ... diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedule_from_partitions.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedule_from_partitions.py index f9c8e90f4415..2da237673f11 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedule_from_partitions.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedule_from_partitions.py @@ -7,8 +7,7 @@ @job(config=my_partitioned_config) -def do_stuff_partitioned(): - ... +def do_stuff_partitioned(): ... do_stuff_partitioned_schedule = build_schedule_from_partitioned_job( @@ -28,8 +27,7 @@ def do_stuff_partitioned(): @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) -def hourly_asset(): - ... +def hourly_asset(): ... partitioned_asset_job = define_asset_job("partitioned_job", selection=[hourly_asset]) diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedules/schedules.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedules/schedules.py index d6c2a44f6f45..3f76e53360cd 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedules/schedules.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/schedules/schedules.py @@ -14,8 +14,7 @@ # start_basic_schedule @job -def my_job(): - ... +def my_job(): ... basic_schedule = ScheduleDefinition(job=my_job, cron_schedule="0 0 * * *") diff --git a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/sensors/sensors.py b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/sensors/sensors.py index 31fd30a42701..7a66c7eca853 100644 --- a/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/sensors/sensors.py +++ b/examples/docs_snippets/docs_snippets/concepts/partitions_schedules_sensors/sensors/sensors.py @@ -78,8 +78,7 @@ def materializes_asset_sensor(): # start_running_in_code @sensor(job=asset_job, default_status=DefaultSensorStatus.RUNNING) -def my_running_sensor(): - ... +def my_running_sensor(): ... # end_running_in_code @@ -204,12 +203,10 @@ def my_s3_sensor(context): @job -def the_job(): - ... +def the_job(): ... -def get_the_db_connection(_): - ... +def get_the_db_connection(_): ... defs = Definitions( diff --git a/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py b/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py index cf55aabe0a2e..20b72ef9f7bb 100644 --- a/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py +++ b/examples/docs_snippets/docs_snippets/concepts/resources/pythonic_resources.py @@ -172,8 +172,7 @@ def new_resource_runtime() -> "Definitions": class DatabaseResource(ConfigurableResource): table: str - def read(self): - ... + def read(self): ... @asset def data_from_database(db_conn: DatabaseResource): @@ -666,12 +665,10 @@ def with_complex_state_example() -> None: class DBConnection: ... - def query(self, body: str): - ... + def query(self, body: str): ... @contextmanager # type: ignore - def get_database_connection(username: str, password: str): - ... + def get_database_connection(username: str, password: str): ... class MyClientResource(ConfigurableResource): username: str @@ -708,8 +705,7 @@ def new_resource_testing_with_state_ops() -> None: class MyClient: ... - def query(self, body: str): - ... + def query(self, body: str): ... class MyClientResource(ConfigurableResource): username: str @@ -757,8 +753,7 @@ def fetch_users(self) -> List[str]: return requests.get(self.url).json() @job - def process_user(): - ... + def process_user(): ... @sensor(job=process_user) def process_new_users_sensor( @@ -821,8 +816,7 @@ def strftime(self, dt: datetime) -> str: return dt.strftime(self.format) @job - def process_data(): - ... + def process_data(): ... @schedule(job=process_data, cron_schedule="* * * * *") def process_data_schedule( diff --git a/examples/docs_snippets/docs_snippets/concepts/resources/resources.py b/examples/docs_snippets/docs_snippets/concepts/resources/resources.py index 51113e9c9d4d..354ce3ef2adf 100644 --- a/examples/docs_snippets/docs_snippets/concepts/resources/resources.py +++ b/examples/docs_snippets/docs_snippets/concepts/resources/resources.py @@ -213,12 +213,10 @@ def use_db_connection(context: OpExecutionContext): @job -def the_job(): - ... +def the_job(): ... -def get_the_db_connection(_): - ... +def get_the_db_connection(_): ... # start_build_resources_example @@ -226,8 +224,7 @@ def get_the_db_connection(_): @resource -def the_credentials(): - ... +def the_credentials(): ... @resource(required_resource_keys={"credentials"}) @@ -263,8 +260,7 @@ def asset_requires_resource(context: AssetExecutionContext): @resource -def foo_resource(): - ... +def foo_resource(): ... # start_asset_provide_resource diff --git a/examples/docs_snippets/docs_snippets/concepts/resources/tests.py b/examples/docs_snippets/docs_snippets/concepts/resources/tests.py index 730a6cd867a6..5ca7a308bd2d 100644 --- a/examples/docs_snippets/docs_snippets/concepts/resources/tests.py +++ b/examples/docs_snippets/docs_snippets/concepts/resources/tests.py @@ -29,8 +29,7 @@ def get_data_without_resource(): class MyApi(ConfigurableResource): - def call(self): - ... + def call(self): ... @op diff --git a/examples/docs_snippets/docs_snippets/deploying/concurrency_limits/concurrency_limits.py b/examples/docs_snippets/docs_snippets/deploying/concurrency_limits/concurrency_limits.py index 4b0eb6e6ca0c..0261a62f58ad 100644 --- a/examples/docs_snippets/docs_snippets/deploying/concurrency_limits/concurrency_limits.py +++ b/examples/docs_snippets/docs_snippets/deploying/concurrency_limits/concurrency_limits.py @@ -4,8 +4,7 @@ @job(tags={"dagster/priority": "3"}) -def important_job(): - ... +def important_job(): ... @schedule( @@ -14,8 +13,7 @@ def important_job(): execution_timezone="US/Central", tags={"dagster/priority": "-1"}, ) -def less_important_schedule(_): - ... +def less_important_schedule(_): ... # end_marker_priority @@ -23,13 +21,11 @@ def less_important_schedule(_): # start_global_concurrency @op(tags={"dagster/concurrency_key": "redshift"}) -def my_redshift_op(): - ... +def my_redshift_op(): ... @asset(op_tags={"dagster/concurrency_key": "redshift"}) -def my_redshift_table(): - ... +def my_redshift_table(): ... # end_global_concurrency @@ -37,13 +33,11 @@ def my_redshift_table(): # start_global_concurrency_priority @op(tags={"dagster/concurrency_key": "foo", "dagster/priority": "3"}) -def my_op(): - ... +def my_op(): ... @asset(op_tags={"dagster/concurrency_key": "foo", "dagster/priority": "3"}) -def my_asset(): - ... +def my_asset(): ... # end_global_concurrency_priority diff --git a/examples/docs_snippets/docs_snippets/deploying/executors/executors.py b/examples/docs_snippets/docs_snippets/deploying/executors/executors.py index 51c8f913a85b..1eb3fe6de32f 100644 --- a/examples/docs_snippets/docs_snippets/deploying/executors/executors.py +++ b/examples/docs_snippets/docs_snippets/deploying/executors/executors.py @@ -6,13 +6,11 @@ # Providing an executor using the job decorator @job(executor_def=multiprocess_executor) -def the_job(): - ... +def the_job(): ... @graph -def the_graph(): - ... +def the_graph(): ... # Providing an executor using graph_def.to_job(...) @@ -34,8 +32,7 @@ def the_asset(): @job -def op_job(): - ... +def op_job(): ... # op_job and asset_job will both use the multiprocess_executor, diff --git a/examples/docs_snippets/docs_snippets/deploying/gcp/gcp_job.py b/examples/docs_snippets/docs_snippets/deploying/gcp/gcp_job.py index 1c64b45c3aa9..a063bd7ed666 100644 --- a/examples/docs_snippets/docs_snippets/deploying/gcp/gcp_job.py +++ b/examples/docs_snippets/docs_snippets/deploying/gcp/gcp_job.py @@ -20,5 +20,4 @@ } }, ) -def gcs_job(): - ... +def gcs_job(): ... diff --git a/examples/docs_snippets/docs_snippets/deploying/monitoring_daemon/run_timeouts.py b/examples/docs_snippets/docs_snippets/deploying/monitoring_daemon/run_timeouts.py index ecfea99e87b9..b4c3ec07396a 100644 --- a/examples/docs_snippets/docs_snippets/deploying/monitoring_daemon/run_timeouts.py +++ b/examples/docs_snippets/docs_snippets/deploying/monitoring_daemon/run_timeouts.py @@ -3,8 +3,7 @@ @job(tags={"dagster/max_runtime": 10}) -def my_job(): - ... +def my_job(): ... asset_job = define_asset_job( diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/assets_ops_graphs/op_graph_asset_input.py b/examples/docs_snippets/docs_snippets/guides/dagster/assets_ops_graphs/op_graph_asset_input.py index 1ab5594b064c..4f4a2f3e0d0f 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/assets_ops_graphs/op_graph_asset_input.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/assets_ops_graphs/op_graph_asset_input.py @@ -2,13 +2,11 @@ @asset -def emails_to_send(): - ... +def emails_to_send(): ... @op -def send_emails(emails) -> None: - ... +def send_emails(emails) -> None: ... @job diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/dagster_pipes/subprocess/with_asset_check/dagster_code.py b/examples/docs_snippets/docs_snippets/guides/dagster/dagster_pipes/subprocess/with_asset_check/dagster_code.py index 215ea1735aa1..54d04c04f237 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/dagster_pipes/subprocess/with_asset_check/dagster_code.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/dagster_pipes/subprocess/with_asset_check/dagster_code.py @@ -13,8 +13,7 @@ @asset -def my_asset(): - ... +def my_asset(): ... @asset_check(asset="my_asset") diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/managing_ml/managing_ml_code.py b/examples/docs_snippets/docs_snippets/guides/dagster/managing_ml/managing_ml_code.py index 0d1f0a0123bf..529934b5048e 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/managing_ml/managing_ml_code.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/managing_ml/managing_ml_code.py @@ -6,15 +6,13 @@ @asset -def my_data(): - ... +def my_data(): ... @asset( auto_materialize_policy=AutoMaterializePolicy.eager(), ) -def my_ml_model(my_data): - ... +def my_ml_model(my_data): ... ## eager_materilization_end @@ -25,16 +23,14 @@ def my_ml_model(my_data): @asset -def my_other_data(): - ... +def my_other_data(): ... @asset( auto_materialize_policy=AutoMaterializePolicy.lazy(), freshness_policy=FreshnessPolicy(maximum_lag_minutes=7 * 24 * 60), ) -def my_other_ml_model(my_other_data): - ... +def my_other_ml_model(my_other_data): ... ## lazy_materlization_end @@ -45,21 +41,18 @@ def my_other_ml_model(my_other_data): @asset -def some_data(): - ... +def some_data(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.lazy()) -def some_ml_model(some_data): - ... +def some_ml_model(some_data): ... @asset( auto_materialize_policy=AutoMaterializePolicy.lazy(), freshness_policy=FreshnessPolicy(maximum_lag_minutes=7 * 24 * 60), ) -def predictions(some_ml_model): - ... +def predictions(some_ml_model): ... ## without_policy_end diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/migrating_to_python_resources_and_config/migrating_resources.py b/examples/docs_snippets/docs_snippets/guides/dagster/migrating_to_python_resources_and_config/migrating_resources.py index 4a1075f4b538..d51c81f1ff38 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/migrating_to_python_resources_and_config/migrating_resources.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/migrating_to_python_resources_and_config/migrating_resources.py @@ -17,8 +17,7 @@ class FancyDbResource: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... @resource(config_schema={"conn_string": str}) def fancy_db_resource(context: InitResourceContext) -> FancyDbResource: @@ -49,8 +48,7 @@ def convert_resource() -> Definitions: class FancyDbResource(ConfigurableResource): conn_string: str - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... # end_convert_resource @@ -81,8 +79,7 @@ def new_style_resource_on_context() -> Definitions: class FancyDbResource(ConfigurableResource): conn_string: str - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... @asset(required_resource_keys={"fancy_db"}) def asset_one(context: AssetExecutionContext) -> None: @@ -105,8 +102,7 @@ def new_style_resource_on_param() -> Definitions: class FancyDbResource(ConfigurableResource): conn_string: str - def execute(self, query: str) -> None: - ... + def execute(self, query: str) -> None: ... # begin_new_style_resource_on_param from dagster import AssetExecutionContext, asset @@ -131,8 +127,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # Alternatively could have been imported from third-party library # from fancy_db import FancyDbClient @@ -159,12 +154,10 @@ def existing_asset(context: AssetExecutionContext) -> None: return defs -def some_expensive_setup() -> None: - ... +def some_expensive_setup() -> None: ... -def some_expensive_teardown() -> None: - ... +def some_expensive_teardown() -> None: ... def old_resource_code_contextmanager() -> Definitions: @@ -172,8 +165,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # begin_old_resource_code_contextmanager @@ -209,8 +201,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # begin_new_resource_code_contextmanager @@ -247,8 +238,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # begin_new_third_party_resource from dagster import AssetExecutionContext, ConfigurableResource, asset @@ -292,8 +282,7 @@ class FancyDbClient: def __init__(self, conn_string: str) -> None: self.conn_string = conn_string - def execute_query(self, query: str) -> None: - ... + def execute_query(self, query: str) -> None: ... # begin_new_third_party_resource_with_interface from dagster import ( diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py b/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py index 57de9e33b3ee..698d910ba451 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/pythonic_config/pythonic_config.py @@ -5,8 +5,7 @@ class Engine: - def execute(self, query: str): - ... + def execute(self, query: str): ... def get_engine(connection_url: str) -> Engine: @@ -123,8 +122,7 @@ class MyDataStructuresConfig(Config): user_scores: Dict[str, int] @asset - def scoreboard(config: MyDataStructuresConfig): - ... + def scoreboard(config: MyDataStructuresConfig): ... result = materialize( [scoreboard], @@ -155,8 +153,7 @@ class MyNestedConfig(Config): user_data: Dict[str, UserData] @asset - def average_age(config: MyNestedConfig): - ... + def average_age(config: MyNestedConfig): ... result = materialize( [average_age], diff --git a/examples/docs_snippets/docs_snippets/guides/dagster/versioning_memoization/memoization_enabled_job.py b/examples/docs_snippets/docs_snippets/guides/dagster/versioning_memoization/memoization_enabled_job.py index ef41a7c152b6..1bc56c179ee4 100644 --- a/examples/docs_snippets/docs_snippets/guides/dagster/versioning_memoization/memoization_enabled_job.py +++ b/examples/docs_snippets/docs_snippets/guides/dagster/versioning_memoization/memoization_enabled_job.py @@ -2,5 +2,4 @@ @job(version_strategy=SourceHashVersionStrategy()) -def the_job(): - ... +def the_job(): ... diff --git a/examples/docs_snippets/docs_snippets/integrations/databricks/databricks.py b/examples/docs_snippets/docs_snippets/integrations/databricks/databricks.py index 9fbc6c4ae50a..ad7f75ebde13 100644 --- a/examples/docs_snippets/docs_snippets/integrations/databricks/databricks.py +++ b/examples/docs_snippets/docs_snippets/integrations/databricks/databricks.py @@ -93,8 +93,7 @@ def scope_schedule_databricks(): from dagster import AssetSelection, asset, define_asset_job, job @asset - def my_databricks_table(): - ... + def my_databricks_table(): ... materialize_databricks_table = define_asset_job( name="materialize_databricks_table", @@ -102,8 +101,7 @@ def my_databricks_table(): ) @job - def my_databricks_job(): - ... + def my_databricks_job(): ... # start_schedule_databricks from dagster import ( diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/dbt.py b/examples/docs_snippets/docs_snippets/integrations/dbt/dbt.py index bfdfb04c76bb..03b094e41f98 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/dbt.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/dbt.py @@ -34,8 +34,7 @@ def scope_schedule_assets_dbt_only(manifest): from dagster_dbt import build_schedule_from_dbt_selection, dbt_assets @dbt_assets(manifest=manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... daily_dbt_assets_schedule = build_schedule_from_dbt_selection( [my_dbt_assets], @@ -52,8 +51,7 @@ def scope_schedule_assets_dbt_and_downstream(manifest): from dagster_dbt import build_dbt_asset_selection, dbt_assets @dbt_assets(manifest=manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... # selects all models tagged with "daily", and all their downstream asset dependencies daily_selection = build_dbt_asset_selection( @@ -73,16 +71,14 @@ def scope_downstream_asset(): from dagster_dbt import dbt_assets @dbt_assets(manifest=MANIFEST_PATH) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - ... + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): ... # start_downstream_asset from dagster_dbt import get_asset_key_for_model from dagster import asset @asset(deps=[get_asset_key_for_model([my_dbt_assets], "my_dbt_model")]) - def my_downstream_asset(): - ... + def my_downstream_asset(): ... # end_downstream_asset_pandas_df_manager @@ -92,8 +88,7 @@ def scope_downstream_asset_pandas_df_manager(): from dagster_dbt import dbt_assets @dbt_assets(manifest=MANIFEST_PATH) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - ... + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): ... # start_downstream_asset_pandas_df_manager from dagster_dbt import get_asset_key_for_model @@ -120,8 +115,7 @@ def scope_upstream_asset(): from dagster_dbt import DbtCliResource, get_asset_key_for_source, dbt_assets @dbt_assets(manifest=MANIFEST_PATH) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - ... + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): ... @asset(key=get_asset_key_for_source([my_dbt_assets], "jaffle_shop")) def orders(): @@ -135,8 +129,7 @@ def scope_upstream_multi_asset(): from dagster_dbt import DbtCliResource, dbt_assets @dbt_assets(manifest=MANIFEST_PATH) - def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): - ... + def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): ... # start_upstream_multi_asset from dagster import multi_asset, AssetOut, Output @@ -163,8 +156,7 @@ def scope_existing_asset(): from dagster import asset @asset - def upstream(): - ... + def upstream(): ... # end_upstream_dagster_asset diff --git a/examples/docs_snippets/docs_snippets/integrations/dbt/potemkin_dag_for_cover_image.py b/examples/docs_snippets/docs_snippets/integrations/dbt/potemkin_dag_for_cover_image.py index a75f78dac334..b7aaeee24334 100644 --- a/examples/docs_snippets/docs_snippets/integrations/dbt/potemkin_dag_for_cover_image.py +++ b/examples/docs_snippets/docs_snippets/integrations/dbt/potemkin_dag_for_cover_image.py @@ -3,6 +3,7 @@ We pull off some dark magic so that generating the screenshot doesn't involve a whole setup with Fivetran and a database. """ + from dagster import asset @@ -10,12 +11,10 @@ class dagster_fivetran: @staticmethod def build_fivetran_assets(connector_id, table_names): @asset(compute_kind="fivetran") - def users(): - ... + def users(): ... @asset(compute_kind="fivetran") - def orders(): - ... + def orders(): ... return [users, orders] @@ -64,8 +63,7 @@ def dbt_project_assets(context: AssetExecutionContext, dbt: DbtCliResource): compute_kind="tensorflow", deps=[get_asset_key_for_model([dbt_project_assets], "daily_order_summary")], ) -def predicted_orders(): - ... +def predicted_orders(): ... # end diff --git a/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_decls.py b/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_decls.py index 2b3224064bca..61fcf871f9db 100644 --- a/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_decls.py +++ b/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_decls.py @@ -5,16 +5,16 @@ def test_docs_snippets_concepts_external_asset_single_decl() -> None: - single_decl_defs: ( - Definitions - ) = docs_snippets.concepts.assets.external_assets.single_declaration.defs + single_decl_defs: Definitions = ( + docs_snippets.concepts.assets.external_assets.single_declaration.defs + ) assert single_decl_defs.get_assets_def("file_in_s3") def test_docs_snippets_concepts_external_asset_external_asset_deps() -> None: - defs_with_deps: ( - Definitions - ) = docs_snippets.concepts.assets.external_assets.external_asset_deps.defs + defs_with_deps: Definitions = ( + docs_snippets.concepts.assets.external_assets.external_asset_deps.defs + ) assert defs_with_deps.get_assets_def("raw_logs") assert defs_with_deps.get_assets_def("processed_logs") assert defs_with_deps.get_assets_def("processed_logs").asset_deps[ diff --git a/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_with_ops.py b/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_with_ops.py index 0d80e6a690c2..66e6b4c5620d 100644 --- a/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_with_ops.py +++ b/examples/docs_snippets/docs_snippets_tests/concepts_tests/external_asset_tests/test_external_assets_with_ops.py @@ -3,9 +3,9 @@ def test_external_assets_update_external_asset_via_op_0() -> None: - defs: ( - Definitions - ) = docs_snippets.concepts.assets.external_assets.update_external_asset_via_op.defs + defs: Definitions = ( + docs_snippets.concepts.assets.external_assets.update_external_asset_via_op.defs + ) a_job_def = defs.get_job_def("a_job") instance = DagsterInstance.ephemeral() result = a_job_def.execute_in_process(instance=instance) diff --git a/examples/docs_snippets/docs_snippets_tests/concepts_tests/partitions_schedules_sensors_tests/backfills_tests/test_single_run_backfill_io_manager.py b/examples/docs_snippets/docs_snippets_tests/concepts_tests/partitions_schedules_sensors_tests/backfills_tests/test_single_run_backfill_io_manager.py index 8be958ae5a81..423d0adf8c5e 100644 --- a/examples/docs_snippets/docs_snippets_tests/concepts_tests/partitions_schedules_sensors_tests/backfills_tests/test_single_run_backfill_io_manager.py +++ b/examples/docs_snippets/docs_snippets_tests/concepts_tests/partitions_schedules_sensors_tests/backfills_tests/test_single_run_backfill_io_manager.py @@ -12,12 +12,10 @@ def test_io_manager(): daily = DailyPartitionsDefinition(start_date="2020-01-01") @asset(partitions_def=daily) - def asset1(): - ... + def asset1(): ... @asset(partitions_def=daily) - def asset2(asset1): - ... + def asset2(asset1): ... assert materialize( [asset1, asset2], diff --git a/examples/docs_snippets/docs_snippets_tests/guides_tests/assets_ops_graphs_tests/test_op_graph_asset_input.py b/examples/docs_snippets/docs_snippets_tests/guides_tests/assets_ops_graphs_tests/test_op_graph_asset_input.py index 590d8a2a53d3..395dc0b09cb3 100644 --- a/examples/docs_snippets/docs_snippets_tests/guides_tests/assets_ops_graphs_tests/test_op_graph_asset_input.py +++ b/examples/docs_snippets/docs_snippets_tests/guides_tests/assets_ops_graphs_tests/test_op_graph_asset_input.py @@ -6,11 +6,9 @@ def test_send_emails_job(): class EmailsIOManager(IOManager): - def load_input(self, context: InputContext): - ... + def load_input(self, context: InputContext): ... - def handle_output(self, context: OutputContext, obj): - ... + def handle_output(self, context: OutputContext, obj): ... send_emails_job.graph.execute_in_process( resources={"io_manager": EmailsIOManager()} diff --git a/examples/with_airflow/with_airflow/airflow_complex_dag.py b/examples/with_airflow/with_airflow/airflow_complex_dag.py index 42c165477795..b0760f5e4e37 100644 --- a/examples/with_airflow/with_airflow/airflow_complex_dag.py +++ b/examples/with_airflow/with_airflow/airflow_complex_dag.py @@ -17,8 +17,7 @@ # specific language governing permissions and limitations # under the License. -"""Example Airflow DAG that shows the complex DAG structure. -""" +"""Example Airflow DAG that shows the complex DAG structure.""" # Type errors ignored because some of these imports target deprecated modules for compatibility with # airflow 1.x and 2.x. diff --git a/examples/with_pyspark_emr/with_pyspark_emr_tests/test_emr_pyspark.py b/examples/with_pyspark_emr/with_pyspark_emr_tests/test_emr_pyspark.py index 75a8388b3b20..c9043eeb995a 100644 --- a/examples/with_pyspark_emr/with_pyspark_emr_tests/test_emr_pyspark.py +++ b/examples/with_pyspark_emr/with_pyspark_emr_tests/test_emr_pyspark.py @@ -1,4 +1,5 @@ """Launching in EMR is prohibitively time consuming, so we just verify that the plan compiles.""" + import os from dagster import materialize_to_memory diff --git a/pyproject.toml b/pyproject.toml index 022cddc9f2f1..01a392d931b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,7 +110,7 @@ extend-exclude = [ line-length = 100 # Fail if Ruff is not running this version. -required-version = "0.2.0" +required-version = "0.3.0" [tool.ruff.lint] diff --git a/python_modules/automation/automation/parse_spark_configs.py b/python_modules/automation/automation/parse_spark_configs.py index b0aed1da96f9..8608cbef1682 100644 --- a/python_modules/automation/automation/parse_spark_configs.py +++ b/python_modules/automation/automation/parse_spark_configs.py @@ -3,6 +3,7 @@ This script parses the Spark configuration parameters downloaded from the Spark Github repository, and codegens a file that contains dagster configurations for these parameters. """ + import re import sys from enum import Enum diff --git a/python_modules/dagster-graphql/dagster_graphql/schema/dagster_types.py b/python_modules/dagster-graphql/dagster_graphql/schema/dagster_types.py index c22c03d4249c..73851305bb35 100644 --- a/python_modules/dagster-graphql/dagster_graphql/schema/dagster_types.py +++ b/python_modules/dagster-graphql/dagster_graphql/schema/dagster_types.py @@ -39,9 +39,9 @@ def to_dagster_type( check.inst_param(pipeline_snapshot, "pipeline_snapshot", JobSnapshot) check.str_param(dagster_type_key, "dagster_type_key") - dagster_type_meta: ( - DagsterTypeSnap - ) = pipeline_snapshot.dagster_type_namespace_snapshot.get_dagster_type_snap(dagster_type_key) + dagster_type_meta: DagsterTypeSnap = ( + pipeline_snapshot.dagster_type_namespace_snapshot.get_dagster_type_snap(dagster_type_key) + ) base_args: Dict[str, Any] = dict( key=dagster_type_meta.key, diff --git a/python_modules/dagster-graphql/dagster_graphql/test/utils.py b/python_modules/dagster-graphql/dagster_graphql/test/utils.py index 7b8f9c296478..aa7b507f04d4 100644 --- a/python_modules/dagster-graphql/dagster_graphql/test/utils.py +++ b/python_modules/dagster-graphql/dagster_graphql/test/utils.py @@ -19,12 +19,10 @@ class GqlResult(Protocol): @property - def data(self) -> Mapping[str, Any]: - ... + def data(self) -> Mapping[str, Any]: ... @property - def errors(self) -> Optional[Sequence[str]]: - ... + def errors(self) -> Optional[Sequence[str]]: ... Selector: TypeAlias = Dict[str, Any] diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_asset_backfill.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_asset_backfill.py index 18704564a2c2..79194e39ccde 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_asset_backfill.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_asset_backfill.py @@ -115,12 +115,10 @@ def get_repo() -> RepositoryDefinition: partitions_def = StaticPartitionsDefinition(["a", "b", "c"]) @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @asset(partitions_def=partitions_def) - def asset2(): - ... + def asset2(): ... @asset() def asset3(): @@ -134,12 +132,10 @@ def get_repo_with_non_partitioned_asset() -> RepositoryDefinition: partitions_def = StaticPartitionsDefinition(["a", "b", "c"]) @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @asset - def asset2(asset1): - ... + def asset2(asset1): ... return Definitions(assets=[asset1, asset2]).get_repository_def() @@ -573,12 +569,10 @@ def test_launch_asset_backfill_with_non_partitioned_asset(): def get_daily_hourly_repo() -> RepositoryDefinition: @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) - def hourly(): - ... + def hourly(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2020-01-01")) - def daily(hourly): - ... + def daily(hourly): ... return Definitions(assets=[hourly, daily]).get_repository_def() @@ -635,16 +629,13 @@ def test_launch_asset_backfill_with_upstream_anchor_asset(): def get_daily_two_hourly_repo() -> RepositoryDefinition: @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) - def hourly1(): - ... + def hourly1(): ... @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) - def hourly2(): - ... + def hourly2(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2020-01-01")) - def daily(hourly1, hourly2): - ... + def daily(hourly1, hourly2): ... return Definitions(assets=[hourly1, hourly2, daily]).get_repository_def() @@ -692,16 +683,13 @@ def test_launch_asset_backfill_with_two_anchor_assets(): def get_daily_hourly_non_partitioned_repo() -> RepositoryDefinition: @asset(partitions_def=HourlyPartitionsDefinition(start_date="2020-01-01-00:00")) - def hourly(): - ... + def hourly(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2020-01-01")) - def daily(hourly): - ... + def daily(hourly): ... @asset - def non_partitioned(hourly): - ... + def non_partitioned(hourly): ... return Definitions(assets=[hourly, daily, non_partitioned]).get_repository_def() diff --git a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_reexecution.py b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_reexecution.py index e0d23b6637c9..8c1c93e1504e 100644 --- a/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_reexecution.py +++ b/python_modules/dagster-graphql/dagster_graphql_tests/graphql/test_reexecution.py @@ -41,9 +41,9 @@ def test_full_pipeline_reexecution_fs_storage(self, graphql_context, snapshot): assert result_one.data["launchPipelineExecution"]["__typename"] == "LaunchRunSuccess" result_one.data["launchPipelineExecution"]["run"]["runId"] = "" - result_one.data["launchPipelineExecution"]["run"][ - "runConfigYaml" - ] = "" + result_one.data["launchPipelineExecution"]["run"]["runConfigYaml"] = ( + "" + ) snapshot.assert_match(result_one.data) @@ -91,9 +91,9 @@ def test_full_pipeline_reexecution_in_memory_storage(self, graphql_context, snap assert result_one.data["launchPipelineExecution"]["__typename"] == "LaunchRunSuccess" result_one.data["launchPipelineExecution"]["run"]["runId"] = "" - result_one.data["launchPipelineExecution"]["run"][ - "runConfigYaml" - ] = "" + result_one.data["launchPipelineExecution"]["run"]["runConfigYaml"] = ( + "" + ) snapshot.assert_match(result_one.data) diff --git a/python_modules/dagster-pipes/dagster_pipes/__init__.py b/python_modules/dagster-pipes/dagster_pipes/__init__.py index 6e5856f50e9e..4cd60e8e1e55 100644 --- a/python_modules/dagster-pipes/dagster_pipes/__init__.py +++ b/python_modules/dagster-pipes/dagster_pipes/__init__.py @@ -562,8 +562,7 @@ def open(self, params: PipesParams) -> Iterator[T_BlobStoreMessageWriterChannel] yield channel @abstractmethod - def make_channel(self, params: PipesParams) -> T_BlobStoreMessageWriterChannel: - ... + def make_channel(self, params: PipesParams) -> T_BlobStoreMessageWriterChannel: ... class PipesBlobStoreMessageWriterChannel(PipesMessageWriterChannel): @@ -584,8 +583,7 @@ def flush_messages(self) -> Sequence[PipesMessage]: return items @abstractmethod - def upload_messages_chunk(self, payload: StringIO, index: int) -> None: - ... + def upload_messages_chunk(self, payload: StringIO, index: int) -> None: ... @contextmanager def buffered_upload_loop(self) -> Iterator[None]: diff --git a/python_modules/dagster-test/dagster_test/benchmarks/partition_stale_status.py b/python_modules/dagster-test/dagster_test/benchmarks/partition_stale_status.py index 539a654ad7d7..d4297a64cf9f 100644 --- a/python_modules/dagster-test/dagster_test/benchmarks/partition_stale_status.py +++ b/python_modules/dagster-test/dagster_test/benchmarks/partition_stale_status.py @@ -120,12 +120,10 @@ def root(context): return {key: randint(0, 100) for key in keys} @asset - def downstream1(root): - ... + def downstream1(root): ... @asset - def downstream2(downstream1): - ... + def downstream2(downstream1): ... all_assets = [root, downstream1, downstream2] with instance_for_test() as instance: diff --git a/python_modules/dagster-test/dagster_test/toys/asset_reconciliation/eager_reconciliation.py b/python_modules/dagster-test/dagster_test/toys/asset_reconciliation/eager_reconciliation.py index 5a49890cf839..42ff5eb0edd8 100644 --- a/python_modules/dagster-test/dagster_test/toys/asset_reconciliation/eager_reconciliation.py +++ b/python_modules/dagster-test/dagster_test/toys/asset_reconciliation/eager_reconciliation.py @@ -7,33 +7,27 @@ @asset -def root1(): - ... +def root1(): ... @asset -def root2(): - ... +def root2(): ... @asset -def diamond_left(root1): - ... +def diamond_left(root1): ... @asset -def diamond_right(root1): - ... +def diamond_right(root1): ... @asset -def diamond_sink(diamond_left, diamond_right): - ... +def diamond_sink(diamond_left, diamond_right): ... @asset -def after_both_roots(root1, root2): - ... +def after_both_roots(root1, root2): ... defs = Definitions( diff --git a/python_modules/dagster-test/dagster_test/toys/basic_assets.py b/python_modules/dagster-test/dagster_test/toys/basic_assets.py index cfa2af2b7bd0..a99245aebee3 100644 --- a/python_modules/dagster-test/dagster_test/toys/basic_assets.py +++ b/python_modules/dagster-test/dagster_test/toys/basic_assets.py @@ -2,23 +2,19 @@ @asset(group_name="basic_assets") -def basic_asset_1(): - ... +def basic_asset_1(): ... @asset(group_name="basic_assets") -def basic_asset_2(basic_asset_1): - ... +def basic_asset_2(basic_asset_1): ... @asset(group_name="basic_assets") -def basic_asset_3(basic_asset_1): - ... +def basic_asset_3(basic_asset_1): ... @asset(group_name="basic_assets") -def basic_asset_4(basic_asset_2, basic_asset_3): - ... +def basic_asset_4(basic_asset_2, basic_asset_3): ... basic_assets_job = define_asset_job( diff --git a/python_modules/dagster-test/dagster_test/toys/data_versions.py b/python_modules/dagster-test/dagster_test/toys/data_versions.py index a4e6be0593eb..0bc193b3f613 100644 --- a/python_modules/dagster-test/dagster_test/toys/data_versions.py +++ b/python_modules/dagster-test/dagster_test/toys/data_versions.py @@ -25,13 +25,11 @@ def observable_same_version(): @asset(code_version="1", deps=[observable_different_version]) -def has_code_version1(context): - ... +def has_code_version1(context): ... @asset(code_version="1", deps=[observable_same_version]) -def has_code_version2(): - ... +def has_code_version2(): ... @asset( @@ -42,13 +40,11 @@ def has_code_version2(): ], code_version="1", ) -def has_code_version_multiple_deps(): - ... +def has_code_version_multiple_deps(): ... @asset(code_version="1", deps=[has_code_version1]) -def downstream_of_code_versioned(): - ... +def downstream_of_code_versioned(): ... @asset @@ -57,8 +53,7 @@ def root_asset_no_code_version(context): @asset(deps=[root_asset_no_code_version]) -def downstream_of_no_code_version(): - ... +def downstream_of_no_code_version(): ... @multi_asset( @@ -74,5 +69,4 @@ def code_versioned_multi_asset(): @asset(deps=["code_versioned_multi_asset2"]) -def downstream_of_code_versioned_multi_asset(): - ... +def downstream_of_code_versioned_multi_asset(): ... diff --git a/python_modules/dagster-test/dagster_test/toys/nothing_input.py b/python_modules/dagster-test/dagster_test/toys/nothing_input.py index d076cd9d8b61..085d00528147 100644 --- a/python_modules/dagster-test/dagster_test/toys/nothing_input.py +++ b/python_modules/dagster-test/dagster_test/toys/nothing_input.py @@ -2,13 +2,11 @@ @op -def op_with_nothing_output() -> None: - ... +def op_with_nothing_output() -> None: ... @op(ins={"in1": In(Nothing)}) -def op_with_nothing_input() -> None: - ... +def op_with_nothing_input() -> None: ... @job diff --git a/python_modules/dagster-test/dagster_test/toys/partition_config.py b/python_modules/dagster-test/dagster_test/toys/partition_config.py index 25891f907ba8..775fb383aa1f 100644 --- a/python_modules/dagster-test/dagster_test/toys/partition_config.py +++ b/python_modules/dagster-test/dagster_test/toys/partition_config.py @@ -7,8 +7,7 @@ def partconf(partition): @op(config_schema={"letter": str}) -def op1(): - ... +def op1(): ... @job(config=partconf) diff --git a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/dynamic_asset_partitions.py b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/dynamic_asset_partitions.py index 4c0dd90050e1..c01fdd1e9481 100644 --- a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/dynamic_asset_partitions.py +++ b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/dynamic_asset_partitions.py @@ -13,13 +13,11 @@ @asset(partitions_def=customers_partitions_def, group_name="dynamic_asset_partitions") -def customers_dynamic_partitions_asset1(): - ... +def customers_dynamic_partitions_asset1(): ... @asset(partitions_def=customers_partitions_def, group_name="dynamic_asset_partitions") -def customers_dynamic_partitions_asset2(customers_dynamic_partitions_asset1): - ... +def customers_dynamic_partitions_asset2(customers_dynamic_partitions_asset1): ... multipartition_w_dynamic_partitions_def = MultiPartitionsDefinition( diff --git a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/failing_partitions.py b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/failing_partitions.py index 0daabb314593..53f1ab325a82 100644 --- a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/failing_partitions.py +++ b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/failing_partitions.py @@ -25,8 +25,7 @@ def failing_static_partitioned(): @asset(partitions_def=StaticPartitionsDefinition(["a", "b", "c"])) -def downstream_of_failing_partitioned(failing_static_partitioned): - ... +def downstream_of_failing_partitioned(failing_static_partitioned): ... time_window_partitions = DailyPartitionsDefinition(start_date="2022-01-01") diff --git a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/hourly_partitions_to_daily.py b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/hourly_partitions_to_daily.py index 6c38d9de1dc1..b8b47f204e0c 100644 --- a/python_modules/dagster-test/dagster_test/toys/partitioned_assets/hourly_partitions_to_daily.py +++ b/python_modules/dagster-test/dagster_test/toys/partitioned_assets/hourly_partitions_to_daily.py @@ -5,15 +5,12 @@ @asset(partitions_def=hourly_partitions_def) -def hourly_asset1() -> None: - ... +def hourly_asset1() -> None: ... @asset(partitions_def=hourly_partitions_def) -def hourly_asset2() -> None: - ... +def hourly_asset2() -> None: ... @asset(partitions_def=daily_partitions_def, deps=[hourly_asset1, hourly_asset2]) -def daily_asset() -> None: - ... +def daily_asset() -> None: ... diff --git a/python_modules/dagster-test/dagster_test/toys/user_computed_data_versions/__init__.py b/python_modules/dagster-test/dagster_test/toys/user_computed_data_versions/__init__.py index 787512e0fc25..e49e9734d563 100644 --- a/python_modules/dagster-test/dagster_test/toys/user_computed_data_versions/__init__.py +++ b/python_modules/dagster-test/dagster_test/toys/user_computed_data_versions/__init__.py @@ -16,6 +16,7 @@ indicating whether a memoized value was used. This information is passed to the Dagster framework by returning a `Nothing` `Output`. """ + import warnings from typing import Sequence, cast diff --git a/python_modules/dagster-webserver/dagster_webserver/graphql.py b/python_modules/dagster-webserver/dagster_webserver/graphql.py index 2bf0f40562b9..60c9f97eaafe 100644 --- a/python_modules/dagster-webserver/dagster_webserver/graphql.py +++ b/python_modules/dagster-webserver/dagster_webserver/graphql.py @@ -61,24 +61,19 @@ def __init__(self, app_path_prefix: str = ""): self._graphql_middleware = self.build_graphql_middleware() @abstractmethod - def build_graphql_schema(self) -> Schema: - ... + def build_graphql_schema(self) -> Schema: ... @abstractmethod - def build_graphql_middleware(self) -> list: - ... + def build_graphql_middleware(self) -> list: ... @abstractmethod - def build_middleware(self) -> List[Middleware]: - ... + def build_middleware(self) -> List[Middleware]: ... @abstractmethod - def build_routes(self) -> List[BaseRoute]: - ... + def build_routes(self) -> List[BaseRoute]: ... @abstractmethod - def make_request_context(self, conn: HTTPConnection): - ... + def make_request_context(self, conn: HTTPConnection): ... def handle_graphql_errors(self, errors: Sequence[GraphQLError]): results = [] diff --git a/python_modules/dagster/dagster/_annotations.py b/python_modules/dagster/dagster/_annotations.py index 374f9fa92d40..70e65f45c454 100644 --- a/python_modules/dagster/dagster/_annotations.py +++ b/python_modules/dagster/dagster/_annotations.py @@ -85,8 +85,7 @@ def deprecated( additional_warn_text: Optional[str] = ..., subject: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> T_Annotatable: - ... +) -> T_Annotatable: ... @overload @@ -97,8 +96,7 @@ def deprecated( additional_warn_text: Optional[str] = ..., subject: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> Callable[[T_Annotatable], T_Annotatable]: - ... +) -> Callable[[T_Annotatable], T_Annotatable]: ... def deprecated( @@ -199,8 +197,7 @@ def deprecated_param( breaking_version: str, additional_warn_text: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> T_Annotatable: - ... +) -> T_Annotatable: ... @overload @@ -211,8 +208,7 @@ def deprecated_param( breaking_version: str, additional_warn_text: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> Callable[[T_Annotatable], T_Annotatable]: - ... +) -> Callable[[T_Annotatable], T_Annotatable]: ... def deprecated_param( @@ -311,8 +307,7 @@ def experimental( additional_warn_text: Optional[str] = ..., subject: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> T_Annotatable: - ... +) -> T_Annotatable: ... @overload @@ -322,8 +317,7 @@ def experimental( additional_warn_text: Optional[str] = ..., subject: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> Callable[[T_Annotatable], T_Annotatable]: - ... +) -> Callable[[T_Annotatable], T_Annotatable]: ... def experimental( @@ -408,8 +402,7 @@ def experimental_param( param: str, additional_warn_text: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> T_Annotatable: - ... +) -> T_Annotatable: ... @overload @@ -419,8 +412,7 @@ def experimental_param( param: str, additional_warn_text: Optional[str] = ..., emit_runtime_warning: bool = ..., -) -> Callable[[T_Annotatable], T_Annotatable]: - ... +) -> Callable[[T_Annotatable], T_Annotatable]: ... def experimental_param( diff --git a/python_modules/dagster/dagster/_check/__init__.py b/python_modules/dagster/dagster/_check/__init__.py index 1be1d4d26586..f113aed85556 100644 --- a/python_modules/dagster/dagster/_check/__init__.py +++ b/python_modules/dagster/dagster/_check/__init__.py @@ -68,8 +68,7 @@ def bool_param(obj: object, param_name: str, additional_message: Optional[str] = @overload def opt_bool_param( obj: object, param_name: str, default: bool, additional_message: Optional[str] = None -) -> bool: - ... +) -> bool: ... @overload @@ -78,8 +77,7 @@ def opt_bool_param( param_name: str, default: Optional[bool] = ..., additional_message: Optional[str] = None, -) -> Optional[bool]: - ... +) -> Optional[bool]: ... def opt_bool_param( @@ -122,15 +120,13 @@ def callable_param( @overload def opt_callable_param( obj: None, param_name: str, default: None = ..., additional_message: Optional[str] = None -) -> None: - ... +) -> None: ... @overload def opt_callable_param( obj: None, param_name: str, default: T_Callable, additional_message: Optional[str] = None -) -> T_Callable: - ... +) -> T_Callable: ... @overload @@ -139,8 +135,7 @@ def opt_callable_param( param_name: str, default: Optional[U_Callable] = ..., additional_message: Optional[str] = None, -) -> T_Callable: - ... +) -> T_Callable: ... def opt_callable_param( @@ -196,8 +191,7 @@ def opt_class_param( default: type, superclass: Optional[type] = None, additional_message: Optional[str] = None, -) -> type: - ... +) -> type: ... @overload @@ -207,8 +201,7 @@ def opt_class_param( default: None = ..., superclass: Optional[type] = None, additional_message: Optional[str] = None, -) -> Optional[type]: - ... +) -> Optional[type]: ... def opt_class_param( @@ -282,8 +275,7 @@ def opt_nullable_dict_param( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = None, -) -> None: - ... +) -> None: ... @overload @@ -293,8 +285,7 @@ def opt_nullable_dict_param( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = None, -) -> Dict: - ... +) -> Dict: ... def opt_nullable_dict_param( @@ -409,8 +400,7 @@ def is_dict( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> Dict[U, V]: - ... +) -> Dict[U, V]: ... @overload @@ -419,8 +409,7 @@ def is_dict( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> Dict[Any, Any]: - ... +) -> Dict[Any, Any]: ... def is_dict( @@ -452,8 +441,7 @@ def float_param(obj: object, param_name: str, additional_message: Optional[str] @overload def opt_float_param( obj: object, param_name: str, default: float, additional_message: Optional[str] = None -) -> float: - ... +) -> float: ... @overload @@ -462,8 +450,7 @@ def opt_float_param( param_name: str, default: Optional[float] = ..., additional_message: Optional[str] = None, -) -> Optional[float]: - ... +) -> Optional[float]: ... def opt_float_param( @@ -564,8 +551,7 @@ def int_param(obj: object, param_name: str, additional_message: Optional[str] = @overload def opt_int_param( obj: object, param_name: str, default: int, additional_message: Optional[str] = ... -) -> int: - ... +) -> int: ... @overload @@ -574,8 +560,7 @@ def opt_int_param( param_name: str, default: Optional[int] = None, additional_message: Optional[str] = None, -) -> Optional[int]: - ... +) -> Optional[int]: ... def opt_int_param( @@ -650,8 +635,7 @@ def opt_inst_param( ttype: TypeOrTupleOfTypes, default: None = ..., additional_message: Optional[str] = None, -) -> Optional[T]: - ... +) -> Optional[T]: ... @overload @@ -661,8 +645,7 @@ def opt_inst_param( ttype: TypeOrTupleOfTypes, default: T, additional_message: Optional[str] = None, -) -> T: - ... +) -> T: ... @overload @@ -672,8 +655,7 @@ def opt_inst_param( ttype: TypeOrTupleOfTypes, default: Optional[T] = ..., additional_message: Optional[str] = None, -) -> T: - ... +) -> T: ... def opt_inst_param( @@ -769,8 +751,7 @@ def opt_nullable_list_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = None, -) -> None: - ... +) -> None: ... @overload @@ -779,8 +760,7 @@ def opt_nullable_list_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = None, -) -> List[T]: - ... +) -> List[T]: ... def opt_nullable_list_param( @@ -948,8 +928,7 @@ def opt_nullable_mapping_param( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> None: - ... +) -> None: ... @overload @@ -959,8 +938,7 @@ def opt_nullable_mapping_param( key_type: Optional[TypeOrTupleOfTypes] = ..., value_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> Mapping[T, U]: - ... +) -> Mapping[T, U]: ... def opt_nullable_mapping_param( @@ -1026,8 +1004,7 @@ def numeric_param( @overload def opt_numeric_param( obj: object, param_name: str, default: Numeric, additional_message: Optional[str] = ... -) -> Numeric: - ... +) -> Numeric: ... @overload @@ -1036,8 +1013,7 @@ def opt_numeric_param( param_name: str, default: Optional[Numeric] = ..., additional_message: Optional[str] = ..., -) -> Optional[Numeric]: - ... +) -> Optional[Numeric]: ... def opt_numeric_param( @@ -1067,8 +1043,7 @@ def path_param( @overload def opt_path_param( obj: None, param_name: str, default: None = ..., additional_message: Optional[str] = ... -) -> None: - ... +) -> None: ... @overload @@ -1077,8 +1052,7 @@ def opt_path_param( param_name: str, default: Union[str, PathLike], additional_message: Optional[str] = ..., -) -> str: - ... +) -> str: ... @overload @@ -1087,8 +1061,7 @@ def opt_path_param( param_name: str, default: Optional[Union[str, PathLike]] = ..., additional_message: Optional[str] = ..., -) -> str: - ... +) -> str: ... def opt_path_param( @@ -1149,8 +1122,7 @@ def opt_nullable_sequence_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> None: - ... +) -> None: ... @overload @@ -1159,8 +1131,7 @@ def opt_nullable_sequence_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = None, additional_message: Optional[str] = ..., -) -> Sequence[T]: - ... +) -> Sequence[T]: ... def opt_nullable_sequence_param( @@ -1259,8 +1230,7 @@ def opt_nullable_set_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> None: - ... +) -> None: ... @overload @@ -1269,8 +1239,7 @@ def opt_nullable_set_param( param_name: str, of_type: Optional[TypeOrTupleOfTypes] = ..., additional_message: Optional[str] = ..., -) -> T_Set: - ... +) -> T_Set: ... def opt_nullable_set_param( @@ -1309,8 +1278,7 @@ def str_param(obj: object, param_name: str, additional_message: Optional[str] = @overload def opt_str_param( obj: object, param_name: str, default: str, additional_message: Optional[str] = ... -) -> str: - ... +) -> str: ... @overload @@ -1319,8 +1287,7 @@ def opt_str_param( param_name: str, default: Optional[str] = ..., additional_message: Optional[str] = ..., -) -> Optional[str]: - ... +) -> Optional[str]: ... def opt_str_param( @@ -1405,8 +1372,7 @@ def opt_tuple_param( of_type: Optional[TypeOrTupleOfTypes] = ..., of_shape: Optional[Tuple[TypeOrTupleOfTypes, ...]] = ..., additional_message: Optional[str] = ..., -) -> Tuple[T, ...]: - ... +) -> Tuple[T, ...]: ... @overload @@ -1416,8 +1382,7 @@ def opt_tuple_param( of_type: Optional[TypeOrTupleOfTypes] = ..., of_shape: Optional[Tuple[TypeOrTupleOfTypes, ...]] = ..., additional_message: Optional[str] = ..., -) -> Tuple[object, ...]: - ... +) -> Tuple[object, ...]: ... def opt_tuple_param( @@ -1452,8 +1417,7 @@ def opt_nullable_tuple_param( of_type: Optional[TypeOrTupleOfTypes] = ..., of_shape: Optional[Tuple[TypeOrTupleOfTypes, ...]] = ..., additional_message: Optional[str] = ..., -) -> None: - ... +) -> None: ... @overload @@ -1463,8 +1427,7 @@ def opt_nullable_tuple_param( of_type: TypeOrTupleOfTypes = ..., of_shape: Optional[Tuple[TypeOrTupleOfTypes, ...]] = ..., additional_message: Optional[str] = None, -) -> Tuple[T, ...]: - ... +) -> Tuple[T, ...]: ... def opt_nullable_tuple_param( diff --git a/python_modules/dagster/dagster/_core/definitions/asset_graph.py b/python_modules/dagster/dagster/_core/definitions/asset_graph.py index 06173ff0bfae..fac7963117c7 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_graph.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_graph.py @@ -70,53 +70,42 @@ class ParentsPartitionsResult(NamedTuple): class AssetGraph(ABC): @property @abstractmethod - def asset_dep_graph(self) -> DependencyGraph[AssetKey]: - ... + def asset_dep_graph(self) -> DependencyGraph[AssetKey]: ... @abstractmethod - def has_asset(self, asset_key: AssetKey) -> bool: - ... + def has_asset(self, asset_key: AssetKey) -> bool: ... @property @abstractmethod - def all_asset_keys(self) -> AbstractSet[AssetKey]: - ... + def all_asset_keys(self) -> AbstractSet[AssetKey]: ... @property @abstractmethod - def materializable_asset_keys(self) -> AbstractSet[AssetKey]: - ... + def materializable_asset_keys(self) -> AbstractSet[AssetKey]: ... @abstractmethod - def is_materializable(self, asset_key: AssetKey) -> bool: - ... + def is_materializable(self, asset_key: AssetKey) -> bool: ... @property @abstractmethod - def observable_asset_keys(self) -> AbstractSet[AssetKey]: - ... + def observable_asset_keys(self) -> AbstractSet[AssetKey]: ... @abstractmethod - def is_observable(self, asset_key: AssetKey) -> bool: - ... + def is_observable(self, asset_key: AssetKey) -> bool: ... @property @abstractmethod - def external_asset_keys(self) -> AbstractSet[AssetKey]: - ... + def external_asset_keys(self) -> AbstractSet[AssetKey]: ... @abstractmethod - def is_external(self, asset_key: AssetKey) -> bool: - ... + def is_external(self, asset_key: AssetKey) -> bool: ... @property @abstractmethod - def executable_asset_keys(self) -> AbstractSet[AssetKey]: - ... + def executable_asset_keys(self) -> AbstractSet[AssetKey]: ... @abstractmethod - def is_executable(self, asset_key: AssetKey) -> bool: - ... + def is_executable(self, asset_key: AssetKey) -> bool: ... @property @cached_method @@ -141,8 +130,7 @@ def toposorted_asset_keys_by_level(self) -> Sequence[AbstractSet[AssetKey]]: ] @abstractmethod - def asset_keys_for_group(self, group_name: str) -> AbstractSet[AssetKey]: - ... + def asset_keys_for_group(self, group_name: str) -> AbstractSet[AssetKey]: ... @functools.cached_property def root_materializable_asset_keys(self) -> AbstractSet[AssetKey]: @@ -160,16 +148,15 @@ def root_executable_asset_keys(self) -> AbstractSet[AssetKey]: @property @abstractmethod - def all_group_names(self) -> AbstractSet[str]: - ... + def all_group_names(self) -> AbstractSet[str]: ... @abstractmethod - def get_partitions_def(self, asset_key: AssetKey) -> Optional[PartitionsDefinition]: - ... + def get_partitions_def(self, asset_key: AssetKey) -> Optional[PartitionsDefinition]: ... @abstractmethod - def get_partition_mappings(self, asset_key: AssetKey) -> Mapping[AssetKey, PartitionMapping]: - ... + def get_partition_mappings( + self, asset_key: AssetKey + ) -> Mapping[AssetKey, PartitionMapping]: ... def get_partition_mapping( self, asset_key: AssetKey, in_asset_key: AssetKey @@ -200,28 +187,24 @@ def is_partitioned(self, asset_key: AssetKey) -> bool: return self.get_partitions_def(asset_key) is not None @abstractmethod - def get_group_name(self, asset_key: AssetKey) -> Optional[str]: - ... + def get_group_name(self, asset_key: AssetKey) -> Optional[str]: ... @abstractmethod - def get_freshness_policy(self, asset_key: AssetKey) -> Optional[FreshnessPolicy]: - ... + def get_freshness_policy(self, asset_key: AssetKey) -> Optional[FreshnessPolicy]: ... @abstractmethod - def get_auto_materialize_policy(self, asset_key: AssetKey) -> Optional[AutoMaterializePolicy]: - ... + def get_auto_materialize_policy( + self, asset_key: AssetKey + ) -> Optional[AutoMaterializePolicy]: ... @abstractmethod - def get_auto_observe_interval_minutes(self, asset_key: AssetKey) -> Optional[float]: - ... + def get_auto_observe_interval_minutes(self, asset_key: AssetKey) -> Optional[float]: ... @abstractmethod - def get_backfill_policy(self, asset_key: AssetKey) -> Optional[BackfillPolicy]: - ... + def get_backfill_policy(self, asset_key: AssetKey) -> Optional[BackfillPolicy]: ... @abstractmethod - def get_code_version(self, asset_key: AssetKey) -> Optional[str]: - ... + def get_code_version(self, asset_key: AssetKey) -> Optional[str]: ... def have_same_partitioning(self, asset_key1: AssetKey, asset_key2: AssetKey) -> bool: """Returns whether the given assets have the same partitions definition.""" @@ -538,8 +521,7 @@ def get_required_multi_asset_keys(self, asset_key: AssetKey) -> AbstractSet[Asse @abstractmethod def get_required_asset_and_check_keys( self, asset_key_or_check_key: AssetKeyOrCheckKey - ) -> AbstractSet[AssetKeyOrCheckKey]: - ... + ) -> AbstractSet[AssetKeyOrCheckKey]: ... @cached_method def get_downstream_freshness_policies( diff --git a/python_modules/dagster/dagster/_core/definitions/asset_graph_subset.py b/python_modules/dagster/dagster/_core/definitions/asset_graph_subset.py index 5c6d3a24322e..ab3e64a79f29 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_graph_subset.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_graph_subset.py @@ -382,12 +382,12 @@ def from_asset_keys( for asset_key in asset_keys: partitions_def = asset_graph.get_partitions_def(asset_key) if partitions_def: - partitions_subsets_by_asset_key[ - asset_key - ] = partitions_def.empty_subset().with_partition_keys( - partitions_def.get_partition_keys( - dynamic_partitions_store=dynamic_partitions_store, - current_time=current_time, + partitions_subsets_by_asset_key[asset_key] = ( + partitions_def.empty_subset().with_partition_keys( + partitions_def.get_partition_keys( + dynamic_partitions_store=dynamic_partitions_store, + current_time=current_time, + ) ) ) else: diff --git a/python_modules/dagster/dagster/_core/definitions/asset_layer.py b/python_modules/dagster/dagster/_core/definitions/asset_layer.py index 8fcaad93f873..35454d9dd684 100644 --- a/python_modules/dagster/dagster/_core/definitions/asset_layer.py +++ b/python_modules/dagster/dagster/_core/definitions/asset_layer.py @@ -237,9 +237,9 @@ def _get_dependency_node_output_handles( ) if curr_node_handle not in outputs_by_graph_handle: - dep_node_output_handles_by_node_output_handle[ - node_output_handle - ] = dependency_node_output_handles + dep_node_output_handles_by_node_output_handle[node_output_handle] = ( + dependency_node_output_handles + ) return dependency_node_output_handles @@ -462,9 +462,9 @@ def from_graph_and_assets_node_mapping( partition_mapping = assets_def.get_partition_mapping_for_input(input_name) if partition_mapping is not None: - partition_mappings_by_asset_dep[ - (node_handle, input_asset_key) - ] = partition_mapping + partition_mappings_by_asset_dep[(node_handle, input_asset_key)] = ( + partition_mapping + ) for output_name, asset_key in assets_def.node_keys_by_output_name.items(): # resolve graph output to the op output it comes from diff --git a/python_modules/dagster/dagster/_core/definitions/assets.py b/python_modules/dagster/dagster/_core/definitions/assets.py index 89d9f45cff56..28488c16d509 100644 --- a/python_modules/dagster/dagster/_core/definitions/assets.py +++ b/python_modules/dagster/dagster/_core/definitions/assets.py @@ -1133,9 +1133,9 @@ def with_attributes( replaced_freshness_policy = self.freshness_policies_by_key.get(key) if replaced_freshness_policy: - replaced_freshness_policies_by_key[ - output_asset_key_replacements.get(key, key) - ] = replaced_freshness_policy + replaced_freshness_policies_by_key[output_asset_key_replacements.get(key, key)] = ( + replaced_freshness_policy + ) if auto_materialize_policy: auto_materialize_policy_conflicts = ( diff --git a/python_modules/dagster/dagster/_core/definitions/assets_job.py b/python_modules/dagster/dagster/_core/definitions/assets_job.py index 5a9883e307f3..a6ac72772216 100644 --- a/python_modules/dagster/dagster/_core/definitions/assets_job.py +++ b/python_modules/dagster/dagster/_core/definitions/assets_job.py @@ -302,9 +302,9 @@ def build_job_partitions_from_assets( if len(assets_with_partitions_defs) == 0: return None - first_asset_with_partitions_def: Union[ - AssetsDefinition, SourceAsset - ] = assets_with_partitions_defs[0] + first_asset_with_partitions_def: Union[AssetsDefinition, SourceAsset] = ( + assets_with_partitions_defs[0] + ) for asset in assets_with_partitions_defs: if asset.partitions_def != first_asset_with_partitions_def.partitions_def: first_asset_key = _key_for_asset(asset).to_string() @@ -345,9 +345,9 @@ def _get_blocking_asset_check_output_handles_by_asset_key( NodeOutputHandle(node_handle, output_name=output_name) ] = check_spec - blocking_asset_check_output_handles_by_asset_key: Dict[ - AssetKey, Set[NodeOutputHandle] - ] = defaultdict(set) + blocking_asset_check_output_handles_by_asset_key: Dict[AssetKey, Set[NodeOutputHandle]] = ( + defaultdict(set) + ) for node_output_handle, check_spec in check_specs_by_node_output_handle.items(): if check_spec.blocking: blocking_asset_check_output_handles_by_asset_key[check_spec.asset_key].add( diff --git a/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule.py b/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule.py index 3b32ad1a1ffe..b4bcbf8314ce 100644 --- a/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule.py +++ b/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule.py @@ -547,12 +547,12 @@ def evaluate_for_asset( else updated_and_will_update_parents ) - updated_parent_assets_by_asset_partition: Dict[ - AssetKeyPartitionKey, Set[AssetKey] - ] = defaultdict(set) - will_update_parent_assets_by_asset_partition: Dict[ - AssetKeyPartitionKey, Set[AssetKey] - ] = defaultdict(set) + updated_parent_assets_by_asset_partition: Dict[AssetKeyPartitionKey, Set[AssetKey]] = ( + defaultdict(set) + ) + will_update_parent_assets_by_asset_partition: Dict[AssetKeyPartitionKey, Set[AssetKey]] = ( + defaultdict(set) + ) for updated_or_will_update_parent in filtered_updated_and_will_update_parents: for child in asset_partitions_by_updated_parents.get(updated_or_will_update_parent, []): diff --git a/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule_evaluation.py b/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule_evaluation.py index 70e6745d59f2..285ad1407cab 100644 --- a/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule_evaluation.py +++ b/python_modules/dagster/dagster/_core/definitions/auto_materialize_rule_evaluation.py @@ -531,30 +531,24 @@ def unpack( @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class FreshnessAutoMaterializeCondition(NamedTuple): - ... +class FreshnessAutoMaterializeCondition(NamedTuple): ... @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class DownstreamFreshnessAutoMaterializeCondition(NamedTuple): - ... +class DownstreamFreshnessAutoMaterializeCondition(NamedTuple): ... @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class ParentMaterializedAutoMaterializeCondition(NamedTuple): - ... +class ParentMaterializedAutoMaterializeCondition(NamedTuple): ... @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class MissingAutoMaterializeCondition(NamedTuple): - ... +class MissingAutoMaterializeCondition(NamedTuple): ... @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class ParentOutdatedAutoMaterializeCondition(NamedTuple): - ... +class ParentOutdatedAutoMaterializeCondition(NamedTuple): ... @whitelist_for_serdes(serializer=BackcompatAutoMaterializeConditionSerializer) -class MaxMaterializationsExceededAutoMaterializeCondition(NamedTuple): - ... +class MaxMaterializationsExceededAutoMaterializeCondition(NamedTuple): ... diff --git a/python_modules/dagster/dagster/_core/definitions/configurable.py b/python_modules/dagster/dagster/_core/definitions/configurable.py index 08f0835cbeb8..de21b63901a7 100644 --- a/python_modules/dagster/dagster/_core/definitions/configurable.py +++ b/python_modules/dagster/dagster/_core/definitions/configurable.py @@ -156,8 +156,7 @@ def copy_for_configured( name: str, description: Optional[str], config_schema: IDefinitionConfigSchema, - ) -> Self: - ... + ) -> Self: ... def _check_configurable_param(configurable: ConfigurableDefinition) -> None: diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py index fbec011f7c82..fd585e76e93e 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/asset_decorator.py @@ -57,8 +57,7 @@ @overload def asset( compute_fn: Callable[..., Any], -) -> AssetsDefinition: - ... +) -> AssetsDefinition: ... @overload @@ -90,8 +89,7 @@ def asset( non_argument_deps: Optional[Union[Set[AssetKey], Set[str]]] = ..., check_specs: Optional[Sequence[AssetCheckSpec]] = ..., owners: Optional[List[str]] = ..., -) -> Callable[[Callable[..., Any]], AssetsDefinition]: - ... +) -> Callable[[Callable[..., Any]], AssetsDefinition]: ... @experimental_param(param="resource_defs") @@ -1017,8 +1015,7 @@ def build_subsettable_asset_ins( @overload def graph_asset( compose_fn: Callable[..., Any], -) -> AssetsDefinition: - ... +) -> AssetsDefinition: ... @overload @@ -1038,8 +1035,7 @@ def graph_asset( resource_defs: Optional[Mapping[str, ResourceDefinition]] = ..., check_specs: Optional[Sequence[AssetCheckSpec]] = None, key: Optional[CoercibleToAssetKey] = None, -) -> Callable[[Callable[..., Any]], AssetsDefinition]: - ... +) -> Callable[[Callable[..., Any]], AssetsDefinition]: ... def graph_asset( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/config_mapping_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/config_mapping_decorator.py index 2d8f9c33e0f8..4316031840c3 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/config_mapping_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/config_mapping_decorator.py @@ -74,8 +74,7 @@ def wrapped_fn(config_as_dict) -> Any: @overload def config_mapping( config_fn: ConfigMappingFn, -) -> ConfigMapping: - ... +) -> ConfigMapping: ... @overload @@ -83,8 +82,7 @@ def config_mapping( *, config_schema: UserConfigSchema = ..., receive_processed_config_values: Optional[bool] = ..., -) -> Callable[[ConfigMappingFn], ConfigMapping]: - ... +) -> Callable[[ConfigMappingFn], ConfigMapping]: ... def config_mapping( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/graph_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/graph_decorator.py index b01b02e90580..997cff7d335a 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/graph_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/graph_decorator.py @@ -103,8 +103,7 @@ def __call__(self, fn: Callable[..., Any]) -> GraphDefinition: @overload -def graph(compose_fn: Callable[..., Any]) -> GraphDefinition: - ... +def graph(compose_fn: Callable[..., Any]) -> GraphDefinition: ... @overload @@ -118,8 +117,7 @@ def graph( out: Optional[Union[GraphOut, Mapping[str, GraphOut]]] = ..., tags: Optional[Mapping[str, Any]] = ..., config: Optional[Union[ConfigMapping, Mapping[str, Any]]] = ..., -) -> _Graph: - ... +) -> _Graph: ... def graph( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/hook_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/hook_decorator.py index c289e3a5d2f1..b756a427745f 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/hook_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/hook_decorator.py @@ -138,8 +138,7 @@ def slack_on_materializations(context, event_list): @overload -def success_hook(hook_fn: SuccessOrFailureHookFn) -> HookDefinition: - ... +def success_hook(hook_fn: SuccessOrFailureHookFn) -> HookDefinition: ... @overload @@ -147,8 +146,7 @@ def success_hook( *, name: Optional[str] = ..., required_resource_keys: Optional[AbstractSet[str]] = ..., -) -> Callable[[SuccessOrFailureHookFn], HookDefinition]: - ... +) -> Callable[[SuccessOrFailureHookFn], HookDefinition]: ... def success_hook( @@ -213,16 +211,14 @@ def _success_hook( @overload -def failure_hook(name: SuccessOrFailureHookFn) -> HookDefinition: - ... +def failure_hook(name: SuccessOrFailureHookFn) -> HookDefinition: ... @overload def failure_hook( name: Optional[str] = ..., required_resource_keys: Optional[AbstractSet[str]] = ..., -) -> Callable[[SuccessOrFailureHookFn], HookDefinition]: - ... +) -> Callable[[SuccessOrFailureHookFn], HookDefinition]: ... def failure_hook( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/job_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/job_decorator.py index c7d51ef73eb6..0f9b42a46004 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/job_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/job_decorator.py @@ -114,8 +114,7 @@ def __call__(self, fn: Callable[..., Any]) -> JobDefinition: @overload -def job(compose_fn: Callable[..., Any]) -> JobDefinition: - ... +def job(compose_fn: Callable[..., Any]) -> JobDefinition: ... @overload @@ -134,8 +133,7 @@ def job( version_strategy: Optional[VersionStrategy] = ..., partitions_def: Optional["PartitionsDefinition"] = ..., input_values: Optional[Mapping[str, object]] = ..., -) -> _Job: - ... +) -> _Job: ... @deprecated_param( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/op_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/op_decorator.py index 15035035e599..95afbe740eb7 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/op_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/op_decorator.py @@ -142,8 +142,7 @@ def __call__(self, fn: Callable[..., Any]) -> "OpDefinition": @overload -def op(compute_fn: Callable[..., Any]) -> "OpDefinition": - ... +def op(compute_fn: Callable[..., Any]) -> "OpDefinition": ... @overload @@ -159,8 +158,7 @@ def op( version: Optional[str] = ..., retry_policy: Optional[RetryPolicy] = ..., code_version: Optional[str] = ..., -) -> _Op: - ... +) -> _Op: ... @deprecated_param( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/repository_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/repository_decorator.py index 659def953696..357e26804f43 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/repository_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/repository_decorator.py @@ -93,14 +93,12 @@ def __call__( Callable[[], Sequence[RepositoryListDefinition]], Callable[[], RepositoryDictSpec], ], - ) -> RepositoryDefinition: - ... + ) -> RepositoryDefinition: ... @overload def __call__( self, fn: Callable[[], Sequence[PendingRepositoryListDefinition]] - ) -> PendingRepositoryDefinition: - ... + ) -> PendingRepositoryDefinition: ... def __call__( self, @@ -221,15 +219,13 @@ def repository( definitions_fn: Union[ Callable[[], Sequence[RepositoryListDefinition]], Callable[[], RepositoryDictSpec] ], -) -> RepositoryDefinition: - ... +) -> RepositoryDefinition: ... @overload def repository( definitions_fn: Callable[..., Sequence[PendingRepositoryListDefinition]], -) -> PendingRepositoryDefinition: - ... +) -> PendingRepositoryDefinition: ... @overload @@ -242,8 +238,7 @@ def repository( default_logger_defs: Optional[Mapping[str, LoggerDefinition]] = ..., _top_level_resources: Optional[Mapping[str, ResourceDefinition]] = ..., _resource_key_mapping: Optional[Mapping[int, str]] = ..., -) -> _Repository: - ... +) -> _Repository: ... def repository( diff --git a/python_modules/dagster/dagster/_core/definitions/decorators/source_asset_decorator.py b/python_modules/dagster/dagster/_core/definitions/decorators/source_asset_decorator.py index 2bc38a7bfcb4..7803922c9e35 100644 --- a/python_modules/dagster/dagster/_core/definitions/decorators/source_asset_decorator.py +++ b/python_modules/dagster/dagster/_core/definitions/decorators/source_asset_decorator.py @@ -30,8 +30,7 @@ @overload -def observable_source_asset(observe_fn: SourceAssetObserveFunction) -> SourceAsset: - ... +def observable_source_asset(observe_fn: SourceAssetObserveFunction) -> SourceAsset: ... @overload @@ -51,8 +50,7 @@ def observable_source_asset( auto_observe_interval_minutes: Optional[float] = None, freshness_policy: Optional[FreshnessPolicy] = None, op_tags: Optional[Mapping[str, Any]] = None, -) -> "_ObservableSourceAsset": - ... +) -> "_ObservableSourceAsset": ... @experimental diff --git a/python_modules/dagster/dagster/_core/definitions/dependency.py b/python_modules/dagster/dagster/_core/definitions/dependency.py index 3b953c9830b1..82bfc1e6af92 100644 --- a/python_modules/dagster/dagster/_core/definitions/dependency.py +++ b/python_modules/dagster/dagster/_core/definitions/dependency.py @@ -240,8 +240,7 @@ def retry_policy(self) -> Optional[RetryPolicy]: return self._retry_policy @abstractmethod - def describe_node(self) -> str: - ... + def describe_node(self) -> str: ... @abstractmethod def get_resource_requirements( @@ -249,8 +248,7 @@ def get_resource_requirements( outer_container: "GraphDefinition", parent_handle: Optional["NodeHandle"] = None, asset_layer: Optional["AssetLayer"] = None, - ) -> Iterator["ResourceRequirement"]: - ... + ) -> Iterator["ResourceRequirement"]: ... class GraphNode(Node): diff --git a/python_modules/dagster/dagster/_core/definitions/executor_definition.py b/python_modules/dagster/dagster/_core/definitions/executor_definition.py index f10c539d06aa..7657dcb00032 100644 --- a/python_modules/dagster/dagster/_core/definitions/executor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/executor_definition.py @@ -195,8 +195,7 @@ def configured( @overload -def executor(name: ExecutorCreationFunction) -> ExecutorDefinition: - ... +def executor(name: ExecutorCreationFunction) -> ExecutorDefinition: ... @overload @@ -206,8 +205,7 @@ def executor( requirements: Optional[ Union[ExecutorRequirementsFunction, Sequence[ExecutorRequirement]] ] = ..., -) -> "_ExecutorDecoratorCallable": - ... +) -> "_ExecutorDecoratorCallable": ... def executor( diff --git a/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py b/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py index bdfb23296e0e..769bcee57c98 100644 --- a/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py +++ b/python_modules/dagster/dagster/_core/definitions/freshness_based_auto_materialize.py @@ -7,6 +7,7 @@ i.e. it`s late enough to pull in the required data time, and early enough to not go over the maximum lag minutes. """ + import datetime from typing import TYPE_CHECKING, AbstractSet, Optional, Sequence, Tuple diff --git a/python_modules/dagster/dagster/_core/definitions/logger_definition.py b/python_modules/dagster/dagster/_core/definitions/logger_definition.py index 4e5e895dee02..76bdd565ebb4 100644 --- a/python_modules/dagster/dagster/_core/definitions/logger_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/logger_definition.py @@ -121,15 +121,13 @@ def copy_for_configured( @overload def logger( config_schema: CoercableToConfigSchema, description: Optional[str] = ... -) -> Callable[["InitLoggerFunction"], "LoggerDefinition"]: - ... +) -> Callable[["InitLoggerFunction"], "LoggerDefinition"]: ... @overload def logger( config_schema: "InitLoggerFunction", description: Optional[str] = ... -) -> "LoggerDefinition": - ... +) -> "LoggerDefinition": ... def logger( diff --git a/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py b/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py index b8a7c5dc69b4..34ed4e5a18c6 100644 --- a/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/multi_asset_sensor_definition.py @@ -140,12 +140,12 @@ def __init__(self, cursor: Optional[str], context: "MultiAssetSensorEvaluationCo break else: partition_key, event_id, trailing_unconsumed_partitioned_event_ids = cursor_list - self._cursor_component_by_asset_key[ - str_asset_key - ] = MultiAssetSensorAssetCursorComponent( - latest_consumed_event_partition=partition_key, - latest_consumed_event_id=event_id, - trailing_unconsumed_partitioned_event_ids=trailing_unconsumed_partitioned_event_ids, + self._cursor_component_by_asset_key[str_asset_key] = ( + MultiAssetSensorAssetCursorComponent( + latest_consumed_event_partition=partition_key, + latest_consumed_event_id=event_id, + trailing_unconsumed_partitioned_event_ids=trailing_unconsumed_partitioned_event_ids, + ) ) self.initial_latest_consumed_event_ids_by_asset_key[str_asset_key] = event_id @@ -419,9 +419,9 @@ def latest_materialization_records_by_key( and record.asset_entry.last_materialization_record.storage_id > (self._get_cursor(record.asset_entry.asset_key).latest_consumed_event_id or 0) ): - asset_event_records[ - record.asset_entry.asset_key - ] = record.asset_entry.last_materialization_record + asset_event_records[record.asset_entry.asset_key] = ( + record.asset_entry.last_materialization_record + ) return asset_event_records @@ -806,9 +806,9 @@ def add_advanced_records( if materialization: self._advanced_record_ids_by_key[asset_key].add(materialization.storage_id) - self._partition_key_by_record_id[ - materialization.storage_id - ] = materialization.partition_key + self._partition_key_by_record_id[materialization.storage_id] = ( + materialization.partition_key + ) def get_cursor_with_advances( self, diff --git a/python_modules/dagster/dagster/_core/definitions/node_definition.py b/python_modules/dagster/dagster/_core/definitions/node_definition.py index 07e3f576cf1f..8a8d973e38a8 100644 --- a/python_modules/dagster/dagster/_core/definitions/node_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/node_definition.py @@ -70,17 +70,14 @@ def __init__( @property @abstractmethod - def node_type_str(self) -> str: - ... + def node_type_str(self) -> str: ... @property @abstractmethod - def is_graph_job_op_node(self) -> bool: - ... + def is_graph_job_op_node(self) -> bool: ... @abstractmethod - def all_dagster_types(self) -> Iterable["DagsterType"]: - ... + def all_dagster_types(self) -> Iterable["DagsterType"]: ... @property def name(self) -> str: @@ -153,24 +150,20 @@ def output_def_named(self, name: str) -> "OutputDefinition": return self._output_dict[name] @abstractmethod - def iterate_node_defs(self) -> Iterable["NodeDefinition"]: - ... + def iterate_node_defs(self) -> Iterable["NodeDefinition"]: ... @abstractmethod - def iterate_op_defs(self) -> Iterable["OpDefinition"]: - ... + def iterate_op_defs(self) -> Iterable["OpDefinition"]: ... @abstractmethod def resolve_output_to_origin( self, output_name: str, handle: Optional["NodeHandle"], - ) -> Tuple["OutputDefinition", Optional["NodeHandle"]]: - ... + ) -> Tuple["OutputDefinition", Optional["NodeHandle"]]: ... @abstractmethod - def resolve_output_to_origin_op_def(self, output_name: str) -> "OpDefinition": - ... + def resolve_output_to_origin_op_def(self, output_name: str) -> "OpDefinition": ... @abstractmethod def resolve_input_to_destinations( @@ -181,16 +174,13 @@ def resolve_input_to_destinations( """ @abstractmethod - def input_has_default(self, input_name: str) -> bool: - ... + def input_has_default(self, input_name: str) -> bool: ... @abstractmethod - def default_value_for_input(self, input_name: str) -> object: - ... + def default_value_for_input(self, input_name: str) -> object: ... @abstractmethod - def input_supports_dynamic_output_dep(self, input_name: str) -> bool: - ... + def input_supports_dynamic_output_dep(self, input_name: str) -> bool: ... def all_input_output_types(self) -> Iterator["DagsterType"]: for input_def in self._input_defs: @@ -237,5 +227,4 @@ def with_retry_policy(self, retry_policy: RetryPolicy) -> "PendingNodeInvocation @abstractmethod def get_inputs_must_be_resolved_top_level( self, asset_layer: "AssetLayer", handle: Optional["NodeHandle"] = None - ) -> Sequence["InputDefinition"]: - ... + ) -> Sequence["InputDefinition"]: ... diff --git a/python_modules/dagster/dagster/_core/definitions/partition.py b/python_modules/dagster/dagster/_core/definitions/partition.py index 85d17199002d..2f15bf7d0574 100644 --- a/python_modules/dagster/dagster/_core/definitions/partition.py +++ b/python_modules/dagster/dagster/_core/definitions/partition.py @@ -940,13 +940,11 @@ def get_partition_keys_not_in_subset( partitions_def: PartitionsDefinition[T_str], current_time: Optional[datetime] = None, dynamic_partitions_store: Optional[DynamicPartitionsStore] = None, - ) -> Iterable[T_str]: - ... + ) -> Iterable[T_str]: ... @abstractmethod @public - def get_partition_keys(self) -> Iterable[T_str]: - ... + def get_partition_keys(self) -> Iterable[T_str]: ... @abstractmethod def get_partition_key_ranges( @@ -954,12 +952,10 @@ def get_partition_key_ranges( partitions_def: PartitionsDefinition, current_time: Optional[datetime] = None, dynamic_partitions_store: Optional[DynamicPartitionsStore] = None, - ) -> Sequence[PartitionKeyRange]: - ... + ) -> Sequence[PartitionKeyRange]: ... @abstractmethod - def with_partition_keys(self, partition_keys: Iterable[str]) -> "PartitionsSubset[T_str]": - ... + def with_partition_keys(self, partition_keys: Iterable[str]) -> "PartitionsSubset[T_str]": ... def with_partition_key_range( self, @@ -993,15 +989,13 @@ def __and__(self, other: "PartitionsSubset") -> "PartitionsSubset[T_str]": ) @abstractmethod - def serialize(self) -> str: - ... + def serialize(self) -> str: ... @classmethod @abstractmethod def from_serialized( cls, partitions_def: PartitionsDefinition[T_str], serialized: str - ) -> "PartitionsSubset[T_str]": - ... + ) -> "PartitionsSubset[T_str]": ... @classmethod @abstractmethod @@ -1011,23 +1005,19 @@ def can_deserialize( serialized: str, serialized_partitions_def_unique_id: Optional[str], serialized_partitions_def_class_name: Optional[str], - ) -> bool: - ... + ) -> bool: ... @abstractmethod - def __len__(self) -> int: - ... + def __len__(self) -> int: ... @abstractmethod - def __contains__(self, value) -> bool: - ... + def __contains__(self, value) -> bool: ... @classmethod @abstractmethod def empty_subset( cls, partitions_def: Optional[PartitionsDefinition] = None - ) -> "PartitionsSubset[T_str]": - ... + ) -> "PartitionsSubset[T_str]": ... def to_serializable_subset(self) -> "PartitionsSubset": return self diff --git a/python_modules/dagster/dagster/_core/definitions/partition_mapping.py b/python_modules/dagster/dagster/_core/definitions/partition_mapping.py index fe3635b53f1f..dee8543981c2 100644 --- a/python_modules/dagster/dagster/_core/definitions/partition_mapping.py +++ b/python_modules/dagster/dagster/_core/definitions/partition_mapping.py @@ -337,8 +337,7 @@ def get_dimension_dependencies( self, upstream_partitions_def: PartitionsDefinition, downstream_partitions_def: PartitionsDefinition, - ) -> Sequence[DimensionDependency]: - ... + ) -> Sequence[DimensionDependency]: ... def get_partitions_def( self, partitions_def: PartitionsDefinition, dimension_name: Optional[str] @@ -376,9 +375,9 @@ def _get_dependency_partitions_subset( # Maps the dimension name and key of a partition in a_partitions_def to the list of # partition keys in b_partitions_def that are dependencies of that partition - dep_b_keys_by_a_dim_and_key: Dict[ - Optional[str], Dict[Optional[str], List[str]] - ] = defaultdict(lambda: defaultdict(list)) + dep_b_keys_by_a_dim_and_key: Dict[Optional[str], Dict[Optional[str], List[str]]] = ( + defaultdict(lambda: defaultdict(list)) + ) required_but_nonexistent_upstream_partitions = set() b_dimension_partitions_def_by_name: Dict[Optional[str], PartitionsDefinition] = ( diff --git a/python_modules/dagster/dagster/_core/definitions/reconstruct.py b/python_modules/dagster/dagster/_core/definitions/reconstruct.py index 8de90f270267..fe0b8e74a4ff 100644 --- a/python_modules/dagster/dagster/_core/definitions/reconstruct.py +++ b/python_modules/dagster/dagster/_core/definitions/reconstruct.py @@ -679,15 +679,13 @@ def job_def_from_pointer(pointer: CodePointer) -> "JobDefinition": def repository_def_from_target_def( target: Union["RepositoryDefinition", "JobDefinition", "GraphDefinition"], repository_load_data: Optional["RepositoryLoadData"] = None, -) -> "RepositoryDefinition": - ... +) -> "RepositoryDefinition": ... @overload def repository_def_from_target_def( target: object, repository_load_data: Optional["RepositoryLoadData"] = None -) -> None: - ... +) -> None: ... def repository_def_from_target_def( diff --git a/python_modules/dagster/dagster/_core/definitions/resource_definition.py b/python_modules/dagster/dagster/_core/definitions/resource_definition.py index d53a76e2ea0f..94f48314c9ab 100644 --- a/python_modules/dagster/dagster/_core/definitions/resource_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/resource_definition.py @@ -359,8 +359,7 @@ def __call__(self, resource_fn: ResourceFunction) -> ResourceDefinition: @overload -def resource(config_schema: ResourceFunction) -> ResourceDefinition: - ... +def resource(config_schema: ResourceFunction) -> ResourceDefinition: ... @overload @@ -369,8 +368,7 @@ def resource( description: Optional[str] = ..., required_resource_keys: Optional[AbstractSet[str]] = ..., version: Optional[str] = ..., -) -> Callable[[ResourceFunction], "ResourceDefinition"]: - ... +) -> Callable[[ResourceFunction], "ResourceDefinition"]: ... def resource( diff --git a/python_modules/dagster/dagster/_core/definitions/run_status_sensor_definition.py b/python_modules/dagster/dagster/_core/definitions/run_status_sensor_definition.py index 82390212c3fa..c7870c1c562f 100644 --- a/python_modules/dagster/dagster/_core/definitions/run_status_sensor_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/run_status_sensor_definition.py @@ -389,8 +389,7 @@ def build_run_status_sensor_context( @overload def run_failure_sensor( name: RunFailureSensorEvaluationFn, -) -> SensorDefinition: - ... +) -> SensorDefinition: ... @overload @@ -430,8 +429,7 @@ def run_failure_sensor( ) -> Callable[ [RunFailureSensorEvaluationFn], SensorDefinition, -]: - ... +]: ... @deprecated_param( diff --git a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py index 014ddf8d69f8..9560627a56e6 100644 --- a/python_modules/dagster/dagster/_core/definitions/schedule_definition.py +++ b/python_modules/dagster/dagster/_core/definitions/schedule_definition.py @@ -608,9 +608,9 @@ def __init__( "to ScheduleDefinition. Must provide only one of the two." ) elif execution_fn: - self._execution_fn: Optional[ - Union[Callable[..., Any], DecoratedScheduleFunction] - ] = None + self._execution_fn: Optional[Union[Callable[..., Any], DecoratedScheduleFunction]] = ( + None + ) if isinstance(execution_fn, DecoratedScheduleFunction): self._execution_fn = execution_fn else: diff --git a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py index bd649af90f3f..7f7801b26f08 100644 --- a/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py +++ b/python_modules/dagster/dagster/_core/definitions/time_window_partitions.py @@ -1625,16 +1625,13 @@ class BaseTimeWindowPartitionsSubset(PartitionsSubset): SERIALIZATION_VERSION = 1 @abstractproperty - def included_time_windows(self) -> Sequence[TimeWindow]: - ... + def included_time_windows(self) -> Sequence[TimeWindow]: ... @abstractproperty - def num_partitions(self) -> int: - ... + def num_partitions(self) -> int: ... @abstractproperty - def partitions_def(self) -> TimeWindowPartitionsDefinition: - ... + def partitions_def(self) -> TimeWindowPartitionsDefinition: ... def _get_partition_time_windows_not_in_subset( self, @@ -1704,22 +1701,18 @@ def get_partition_keys_not_in_subset( return partition_keys @abstractproperty - def first_start(self) -> datetime: - ... + def first_start(self) -> datetime: ... @abstractproperty - def is_empty(self) -> bool: - ... + def is_empty(self) -> bool: ... @abstractmethod - def cheap_ends_before(self, dt: datetime, dt_cron_schedule: str) -> bool: - ... + def cheap_ends_before(self, dt: datetime, dt_cron_schedule: str) -> bool: ... @abstractmethod def with_partitions_def( self, partitions_def: TimeWindowPartitionsDefinition - ) -> "BaseTimeWindowPartitionsSubset": - ... + ) -> "BaseTimeWindowPartitionsSubset": ... def get_partition_key_ranges( self, diff --git a/python_modules/dagster/dagster/_core/events/__init__.py b/python_modules/dagster/dagster/_core/events/__init__.py index d1dddd0ffc9f..a89f5b8ae3de 100644 --- a/python_modules/dagster/dagster/_core/events/__init__.py +++ b/python_modules/dagster/dagster/_core/events/__init__.py @@ -1,4 +1,5 @@ """Structured representations of system events.""" + import logging import os import sys diff --git a/python_modules/dagster/dagster/_core/execution/context/system.py b/python_modules/dagster/dagster/_core/execution/context/system.py index d698765c1cdf..cef218fb8ccd 100644 --- a/python_modules/dagster/dagster/_core/execution/context/system.py +++ b/python_modules/dagster/dagster/_core/execution/context/system.py @@ -3,6 +3,7 @@ so we have a different layer of objects that encode the explicit public API in the user_context module. """ + from abc import ABC, abstractmethod from dataclasses import dataclass from hashlib import sha256 diff --git a/python_modules/dagster/dagster/_core/execution/execution_result.py b/python_modules/dagster/dagster/_core/execution/execution_result.py index f7f643f06bf1..e54891520767 100644 --- a/python_modules/dagster/dagster/_core/execution/execution_result.py +++ b/python_modules/dagster/dagster/_core/execution/execution_result.py @@ -22,18 +22,15 @@ class ExecutionResult(ABC): @property @abstractmethod - def job_def(self) -> JobDefinition: - ... + def job_def(self) -> JobDefinition: ... @property @abstractmethod - def dagster_run(self) -> DagsterRun: - ... + def dagster_run(self) -> DagsterRun: ... @property @abstractmethod - def all_events(self) -> Sequence[DagsterEvent]: - ... + def all_events(self) -> Sequence[DagsterEvent]: ... @property @abstractmethod diff --git a/python_modules/dagster/dagster/_core/execution/plan/inputs.py b/python_modules/dagster/dagster/_core/execution/plan/inputs.py index bbea78df5a08..09794e6c7c37 100644 --- a/python_modules/dagster/dagster/_core/execution/plan/inputs.py +++ b/python_modules/dagster/dagster/_core/execution/plan/inputs.py @@ -105,8 +105,7 @@ def step_output_handle_dependencies(self) -> Sequence[StepOutputHandle]: @abstractmethod def load_input_object( self, step_context: "StepExecutionContext", input_def: InputDefinition - ) -> Iterator[object]: - ... + ) -> Iterator[object]: ... def required_resource_keys(self, _job_def: JobDefinition) -> AbstractSet[str]: return set() diff --git a/python_modules/dagster/dagster/_core/execution/plan/plan.py b/python_modules/dagster/dagster/_core/execution/plan/plan.py index 528b041ee2da..f15e7219bbcd 100644 --- a/python_modules/dagster/dagster/_core/execution/plan/plan.py +++ b/python_modules/dagster/dagster/_core/execution/plan/plan.py @@ -377,9 +377,9 @@ def _build_from_sorted_nodes( ) step = self.get_step_by_node_handle(check.not_none(resolved_handle)) if isinstance(step, (ExecutionStep, UnresolvedCollectExecutionStep)): - step_output_handle: Union[ - StepOutputHandle, UnresolvedStepOutputHandle - ] = StepOutputHandle(step.key, resolved_output_def.name) + step_output_handle: Union[StepOutputHandle, UnresolvedStepOutputHandle] = ( + StepOutputHandle(step.key, resolved_output_def.name) + ) elif isinstance(step, UnresolvedMappedExecutionStep): step_output_handle = UnresolvedStepOutputHandle( step.handle, @@ -1459,9 +1459,9 @@ def _compute_step_maps( past_mappings = known_state.dynamic_mappings if known_state else {} executable_map: Dict[str, Union[StepHandle, ResolvedFromDynamicStepHandle]] = {} - resolvable_map: Dict[ - FrozenSet[str], List[Union[StepHandle, UnresolvedStepHandle]] - ] = defaultdict(list) + resolvable_map: Dict[FrozenSet[str], List[Union[StepHandle, UnresolvedStepHandle]]] = ( + defaultdict(list) + ) for handle in step_handles_to_execute: step = step_dict[handle] if isinstance(step, ExecutionStep): diff --git a/python_modules/dagster/dagster/_core/executor/child_process_executor.py b/python_modules/dagster/dagster/_core/executor/child_process_executor.py index ed7f5ab6970c..c2de8911431b 100644 --- a/python_modules/dagster/dagster/_core/executor/child_process_executor.py +++ b/python_modules/dagster/dagster/_core/executor/child_process_executor.py @@ -1,6 +1,5 @@ """Facilities for running arbitrary commands in child processes.""" - import os import queue import sys diff --git a/python_modules/dagster/dagster/_core/host_representation/code_location.py b/python_modules/dagster/dagster/_core/host_representation/code_location.py index 1d2b177512e9..870e139226ef 100644 --- a/python_modules/dagster/dagster/_core/host_representation/code_location.py +++ b/python_modules/dagster/dagster/_core/host_representation/code_location.py @@ -310,8 +310,7 @@ def get_repository_python_origin(self, repository_name: str) -> "RepositoryPytho ) @abstractmethod - def get_dagster_library_versions(self) -> Optional[Mapping[str, str]]: - ... + def get_dagster_library_versions(self) -> Optional[Mapping[str, str]]: ... class InProcessCodeLocation(CodeLocation): diff --git a/python_modules/dagster/dagster/_core/host_representation/external_data.py b/python_modules/dagster/dagster/_core/host_representation/external_data.py index 96d2109039f1..ca7801c707db 100644 --- a/python_modules/dagster/dagster/_core/host_representation/external_data.py +++ b/python_modules/dagster/dagster/_core/host_representation/external_data.py @@ -3,6 +3,7 @@ business logic or clever indexing. Use the classes in external.py for that. """ + import inspect import json from abc import ABC, abstractmethod @@ -660,8 +661,7 @@ def __new__(cls, error: Optional[SerializableErrorInfo]): class ExternalPartitionsDefinitionData(ABC): @abstractmethod - def get_partitions_definition(self) -> PartitionsDefinition: - ... + def get_partitions_definition(self) -> PartitionsDefinition: ... @whitelist_for_serdes @@ -1565,9 +1565,9 @@ def external_asset_nodes_from_defs( job_defs: Sequence[JobDefinition], assets_defs_by_key: Mapping[AssetKey, AssetsDefinition], ) -> Sequence[ExternalAssetNode]: - node_defs_by_asset_key: Dict[ - AssetKey, List[Tuple[NodeOutputHandle, JobDefinition]] - ] = defaultdict(list) + node_defs_by_asset_key: Dict[AssetKey, List[Tuple[NodeOutputHandle, JobDefinition]]] = ( + defaultdict(list) + ) asset_info_by_asset_key: Dict[AssetKey, AssetOutputInfo] = dict() freshness_policy_by_asset_key: Dict[AssetKey, FreshnessPolicy] = dict() metadata_by_asset_key: Dict[AssetKey, RawMetadataMapping] = dict() diff --git a/python_modules/dagster/dagster/_core/host_representation/represented.py b/python_modules/dagster/dagster/_core/host_representation/represented.py index adb8d5d56b98..332659edb457 100644 --- a/python_modules/dagster/dagster/_core/host_representation/represented.py +++ b/python_modules/dagster/dagster/_core/host_representation/represented.py @@ -22,8 +22,7 @@ class RepresentedJob(ABC): @property @abstractmethod - def _job_index(self) -> JobIndex: - ... + def _job_index(self) -> JobIndex: ... @property def name(self) -> str: diff --git a/python_modules/dagster/dagster/_core/instance/__init__.py b/python_modules/dagster/dagster/_core/instance/__init__.py index e43befbe8f88..701c7bf336ad 100644 --- a/python_modules/dagster/dagster/_core/instance/__init__.py +++ b/python_modules/dagster/dagster/_core/instance/__init__.py @@ -308,12 +308,10 @@ def register_instance(self, instance: T_DagsterInstance) -> None: @runtime_checkable class DynamicPartitionsStore(Protocol): @abstractmethod - def get_dynamic_partitions(self, partitions_def_name: str) -> Sequence[str]: - ... + def get_dynamic_partitions(self, partitions_def_name: str) -> Sequence[str]: ... @abstractmethod - def has_dynamic_partition(self, partitions_def_name: str, partition_key: str) -> bool: - ... + def has_dynamic_partition(self, partitions_def_name: str, partition_key: str) -> bool: ... class DagsterInstance(DynamicPartitionsStore): @@ -369,9 +367,9 @@ class DagsterInstance(DynamicPartitionsStore): # Stores TemporaryDirectory instances that were created for DagsterInstance.local_temp() calls # to be removed once the instance is garbage collected. - _TEMP_DIRS: ( - "weakref.WeakKeyDictionary[DagsterInstance, TemporaryDirectory]" - ) = weakref.WeakKeyDictionary() + _TEMP_DIRS: "weakref.WeakKeyDictionary[DagsterInstance, TemporaryDirectory]" = ( + weakref.WeakKeyDictionary() + ) def __init__( self, diff --git a/python_modules/dagster/dagster/_core/pipes/utils.py b/python_modules/dagster/dagster/_core/pipes/utils.py index b6a870b340ee..7041c195ab85 100644 --- a/python_modules/dagster/dagster/_core/pipes/utils.py +++ b/python_modules/dagster/dagster/_core/pipes/utils.py @@ -335,8 +335,7 @@ def get_params(self) -> Iterator[PipesParams]: """ @abstractmethod - def download_messages_chunk(self, index: int, params: PipesParams) -> Optional[str]: - ... + def download_messages_chunk(self, index: int, params: PipesParams) -> Optional[str]: ... def _messages_thread( self, @@ -438,20 +437,16 @@ def _logs_thread( class PipesLogReader(ABC): @abstractmethod - def start(self, params: PipesParams, is_session_closed: Event) -> None: - ... + def start(self, params: PipesParams, is_session_closed: Event) -> None: ... @abstractmethod - def stop(self) -> None: - ... + def stop(self) -> None: ... @abstractmethod - def is_running(self) -> bool: - ... + def is_running(self) -> bool: ... @abstractmethod - def target_is_readable(self, params: PipesParams) -> bool: - ... + def target_is_readable(self, params: PipesParams) -> bool: ... @property def name(self) -> str: @@ -474,8 +469,7 @@ def __init__(self, *, interval: float = 10, target_stream: TextIO): self.thread: Optional[Thread] = None @abstractmethod - def download_log_chunk(self, params: PipesParams) -> Optional[str]: - ... + def download_log_chunk(self, params: PipesParams) -> Optional[str]: ... def start(self, params: PipesParams, is_session_closed: Event) -> None: self.thread = Thread(target=self._reader_thread, args=(params, is_session_closed)) diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/001_initial_schedule.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/001_initial_schedule.py index c7b800b7a4e8..c93db974d759 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/001_initial_schedule.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/001_initial_schedule.py @@ -5,6 +5,7 @@ Create Date: 2019-11-22 11:02:06.180581 """ + # revision identifiers, used by Alembic. revision = "da7cd32b690d" down_revision = "567bc23fd1ac" diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_postgres.py index 865e4d139f77..f58098496299 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-02-10 12:52:49.540462 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_sqlite.py index f18a0fab2025..a195772c882b 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/002_cascade_run_deletion_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-02-10 18:13:58.993653 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_postgres.py index 1dbe06ef3583..2118f77ee7e6 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-03-31 11:21:26.811734 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_sqlite.py index cfb5c930fc24..684dcc507d7a 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/003_add_step_key_pipeline_name_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-03-31 11:01:42.609069 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/004_add_snapshots_to_run_storage.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/004_add_snapshots_to_run_storage.py index 7cb0b8001832..ba993ed40cad 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/004_add_snapshots_to_run_storage.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/004_add_snapshots_to_run_storage.py @@ -5,6 +5,7 @@ Create Date: 2020-04-09 05:57:20.639458 """ + import sqlalchemy as sa from alembic import op from dagster._core.storage.migration.utils import has_column, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_postgres.py index eddc617926d4..eaac8ec43dcd 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-04-28 09:17:33.253185 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_sqlite.py index f0c72e4e8414..73c9b56e75bc 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/005_add_asset_key_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-04-28 09:35:54.768791 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/006_scheduler_update_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/006_scheduler_update_postgres.py index eba4137037a5..9351477a6a45 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/006_scheduler_update_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/006_scheduler_update_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-06-10 10:00:57.793622 """ + from alembic import op from dagster._core.storage.migration.utils import get_currently_upgrading_instance, has_table from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_postgres.py index d829debb8f85..13f1d182e0d3 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-06-11 09:27:11.922143 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_sqlite.py index 160742a5aaa4..3e3f0e58fbf6 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/007_create_run_id_idx_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-06-11 10:40:25.216776 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_postgres.py index 99d7530d9998..3762513c8381 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-12-01 12:19:34.460760 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_sqlite.py index 1f59556795a0..fedca4e68bee 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/008_add_run_tags_index_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-12-01 12:10:23.650381 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_postgres.py index b541b660b4fb..15194008c282 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_postgres.py @@ -5,6 +5,7 @@ Create Date: 2020-12-21 10:13:54.430623 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_sqlite.py index 967605a3a6a6..d7d2819ded40 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/009_add_partition_column_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2020-12-21 10:07:10.099687 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_postgres.py index 0bd148740521..4d719e06f41e 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-05 15:21:52.820686 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_sqlite.py index 155c842e5922..1ff0a15c2e29 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/010_add_run_partition_columns_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-01-05 14:39:50.395455 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_postgres.py index 539df0aed1c1..bad66647700f 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-11 22:20:01.253271 """ + from alembic import op from dagster._core.storage.migration.utils import get_currently_upgrading_instance, has_table from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_1.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_1.py index 9184085e7bfe..d8445ae872b6 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_1.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_1.py @@ -5,6 +5,7 @@ Create Date: 2020-06-10 09:05:47.963960 """ + from alembic import op from dagster._core.storage.migration.utils import get_currently_upgrading_instance, has_table from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_2.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_2.py index 9b2bf6f95d82..897b25b6f4b0 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_2.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/011_wipe_schedules_table_for_0_10_0_sqlite_2.py @@ -5,6 +5,7 @@ Create Date: 2021-01-11 22:16:50.896040 """ + from alembic import op from dagster._core.storage.migration.utils import get_currently_upgrading_instance, has_table from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_postgres.py index 4e4f51c81cc8..4ce3a1959d91 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 14:42:51.215325 """ + from dagster._core.storage.migration.utils import create_0_10_0_run_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_sqlite.py index b765fa6763c7..13d809fc04f0 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/012_0_10_0_create_new_run_tables_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 10:48:24.022054 """ + from dagster._core.storage.migration.utils import create_0_10_0_run_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_postgres.py index 196dfcce5cd3..a10cfb124e6d 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 14:42:57.878627 """ + from dagster._core.storage.migration.utils import create_0_10_0_event_log_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_sqlite.py index 1a9f64dd559a..b3b0a380e3e3 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/013_0_10_0_create_new_event_log_tables_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 12:54:27.921898 """ + from dagster._core.storage.migration.utils import create_0_10_0_event_log_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_postgres.py index 8d6068778790..c441c7c8f1b2 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 14:43:03.678784 """ + from dagster._core.storage.migration.utils import create_0_10_0_schedule_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_sqlite.py index 98950f761388..196e03af6268 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/014_0_10_0_create_new_schedule_tables_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-01-13 12:56:41.971500 """ + from dagster._core.storage.migration.utils import create_0_10_0_schedule_tables # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_postgres.py index d3c512f262aa..344a9cf05c39 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-01-14 12:39:53.493651 """ + import sqlalchemy as sa from alembic import op from dagster._core.storage.migration.utils import has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_sqlite.py index bc0e22b1b411..8b5d519b521c 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/015_change_varchar_to_text_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-01-14 12:29:33.410870 """ + import sqlalchemy as sa from alembic import op from dagster._core.storage.migration.utils import has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_postgres.py index 64000d38ba42..5c8902bc25cd 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-02-10 14:58:02.954242 """ + from dagster._core.storage.migration.utils import create_bulk_actions_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_sqlite.py index e3d9c4c20961..468d94fcfdb6 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/016_add_bulk_actions_table_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-02-10 14:45:29.022887 """ + from dagster._core.storage.migration.utils import create_bulk_actions_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_postgres.py index 8126a4479a8f..270e753043ad 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-02-23 16:00:45.689578 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_sqlite.py index d1f94a693c47..c0c53b6ed52e 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/017_add_run_status_index_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-02-23 15:55:33.837945 """ + from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_mysql.py index f32208e4b8a8..30e0a8f59678 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_mysql.py @@ -5,6 +5,7 @@ Create Date: 2021-03-15 15:08:10.277993 """ + from dagster._core.storage.migration.utils import add_asset_materialization_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_postgres.py index 5d14cbcf1c00..3c70eb2a49ba 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-03-04 10:20:27.847208 """ + from dagster._core.storage.migration.utils import add_asset_materialization_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_sqlite.py index fa5c812dc153..6e37b2130f8c 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/018_add_asset_tags_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-03-04 10:01:01.877875 """ + from dagster._core.storage.migration.utils import add_asset_materialization_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_postgres.py index 618918b4d444..6b3750edcb4b 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-03-11 14:59:25.755063 """ + import sqlalchemy as sa from alembic import op from dagster._core.storage.migration.utils import has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_sqlite.py index 5133e4b0f255..7ceb2f3ecdeb 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/019_0_11_0_db_text_to_db_string_unique_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-03-11 15:02:29.174707 """ + import sqlalchemy as sa from alembic import op from dagster._core.storage.migration.utils import has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_mysql.py index 8aad31bfdc2e..8c6aadcee969 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_mysql.py @@ -5,6 +5,7 @@ Create Date: 2021-03-17 16:41:14.457324 """ + from dagster._core.storage.migration.utils import add_asset_details_column # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_postgres.py index 49c7af2e41dd..4ae79b78a9a7 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-03-17 16:40:52.449012 """ + from dagster._core.storage.migration.utils import add_asset_details_column # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_sqlite.py index 71db341a539a..e3a841fbfcc0 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/020_add_column_asset_body_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-03-17 16:38:09.418235 """ + from dagster._core.storage.migration.utils import add_asset_details_column # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_mysql.py index 7b7f9e0e94fd..3e5db0ae0065 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_mysql.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 14:04:04.808944 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_postgres.py index b4d5d888c77a..8b589154a6f1 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 13:56:08.987201 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_sqlite.py index 72ae9be615db..1effdb8ea096 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/021_add_column_mode_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 13:49:53.668345 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_mysql.py index 73cb759ca3a4..9d96391b7070 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_mysql.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 10:53:45.164780 """ + from dagster._core.storage.migration.utils import extract_asset_keys_idx_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_postgres.py index 57d714e04150..76ae4e7d8daf 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 10:52:50.862728 """ + from dagster._core.storage.migration.utils import extract_asset_keys_idx_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_sqlite.py index fc7e30914dac..d8d509bf793b 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/022_extract_asset_keys_index_columns_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-07-06 10:51:26.269010 """ + from dagster._core.storage.migration.utils import extract_asset_keys_idx_columns # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_mysql.py index d569844c8c7d..005c3cf836a0 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_mysql.py @@ -5,6 +5,7 @@ Create Date: 2021-09-08 10:29:08.823969 """ + from dagster._core.storage.migration.utils import create_event_log_event_idx # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_postgres.py index e82bf354d314..3beef9d97a71 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-09-08 10:28:28.730620 """ + from dagster._core.storage.migration.utils import create_event_log_event_idx # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_sqlite.py index a5bcb2df7d2f..3321b6bc6c3c 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/023_add_event_log_event_type_idx_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-09-08 10:42:42.063814 """ + from dagster._core.storage.migration.utils import create_event_log_event_idx # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_mysql.py index 9b8e5a9cde91..66b17cfc971e 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-01-25 09:26:35.820814 """ + from dagster._core.storage.migration.utils import ( add_run_record_start_end_timestamps, drop_run_record_start_end_timestamps, diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_postgres.py index 904c73fa680f..4b2ae2cc9d3c 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_postgres.py @@ -5,6 +5,7 @@ Create Date: 2021-12-20 13:41:14.924529 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_sqlite.py index 350204a8a651..e2d3093f1ec4 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/024_add_columns_start_time_and_end_time_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2021-12-20 13:18:31.122983 """ + import sqlalchemy as sa from alembic import op from sqlalchemy import inspect diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_mysql.py index 399f24d7473b..bf9019e22057 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-01-20 11:45:26.092743 """ + from dagster._core.storage.migration.utils import create_run_range_indices # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_postgres.py index 9d449038b18d..8703530b4e6a 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_postgres.py @@ -5,6 +5,7 @@ Create Date: 2022-01-20 11:43:21.070463 """ + from dagster._core.storage.migration.utils import create_run_range_indices # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_sqlite.py index 3261fd3405e2..eeb7b3fb6fb1 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/025_add_range_index_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2022-01-20 11:39:54.203976 """ + from dagster._core.storage.migration.utils import create_run_range_indices # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/026_convert_start_end_times_format_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/026_convert_start_end_times_format_mysql.py index d2dac1cc5c54..0e8520dd4074 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/026_convert_start_end_times_format_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/026_convert_start_end_times_format_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-02-01 15:21:22.257972 """ + from alembic import op from dagster._core.storage.migration.utils import ( add_run_record_start_end_timestamps, diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_mysql.py index 5517200ad8b2..3a1835e6879e 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-03-23 13:00:19.789472 """ + from dagster._core.storage.migration.utils import create_schedule_secondary_index_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_postgres.py index e7267f617763..3326f10d8b6f 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_postgres.py @@ -5,6 +5,7 @@ Create Date: 2022-03-23 12:59:59.571272 """ + from dagster._core.storage.migration.utils import create_schedule_secondary_index_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_sqlite.py index 957c9215a7d0..b60e2c169287 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/027_add_migration_table_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2022-03-23 12:58:43.144576 """ + from dagster._core.storage.migration.utils import create_schedule_secondary_index_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_mysql.py index 9f8ed573fb44..32ca56b42b35 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-03-18 16:17:20.338259 """ + from dagster._core.storage.migration.utils import create_instigators_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_postgres.py index f7745cc5c480..2f08e5161406 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_postgres.py @@ -5,6 +5,7 @@ Create Date: 2022-03-18 16:17:06.516027 """ + from dagster._core.storage.migration.utils import create_instigators_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_sqlite.py index 1e5c3e1fa7f9..378e3caa5c92 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/028_add_instigators_table_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2022-03-18 16:16:21.007430 """ + from dagster._core.storage.migration.utils import create_instigators_table # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_mysql.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_mysql.py index e450ba74c378..2e01cbfd8f62 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_mysql.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_mysql.py @@ -5,6 +5,7 @@ Create Date: 2022-03-25 10:29:10.895341 """ + from dagster._core.storage.migration.utils import create_tick_selector_index # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_postgres.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_postgres.py index 96cc0b1486cd..26a9bc8b0d71 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_postgres.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_postgres.py @@ -5,6 +5,7 @@ Create Date: 2022-03-25 10:28:53.372766 """ + from dagster._core.storage.migration.utils import create_tick_selector_index # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_sqlite.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_sqlite.py index 65ed9827af60..a67d1e417dd7 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_sqlite.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/029_add_tick_selector_index_sqlite.py @@ -5,6 +5,7 @@ Create Date: 2022-03-25 10:28:29.065161 """ + from dagster._core.storage.migration.utils import create_tick_selector_index # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/030_add_columns_action_type_and_selector_id_.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/030_add_columns_action_type_and_selector_id_.py index fa1c7902e6b1..c647bf2c5718 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/030_add_columns_action_type_and_selector_id_.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/030_add_columns_action_type_and_selector_id_.py @@ -5,6 +5,7 @@ Create Date: 2022-05-20 15:00:01.260860 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_column, has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/031_add_kvs_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/031_add_kvs_table.py index 3492ef0b6739..0faf3d1cb588 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/031_add_kvs_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/031_add_kvs_table.py @@ -5,6 +5,7 @@ Create Date: 2022-06-06 15:48:51.559562 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/032_rebuild_event_indexes.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/032_rebuild_event_indexes.py index e07af377be11..0e133f97ea86 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/032_rebuild_event_indexes.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/032_rebuild_event_indexes.py @@ -5,6 +5,7 @@ Create Date: 2022-10-19 13:33:02.540229 """ + from dagster._core.storage.migration.utils import ( add_id_based_event_indices, drop_id_based_event_indices, diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/033_add_asset_event_tags_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/033_add_asset_event_tags_table.py index e9f293fcb5c9..ba5aae309195 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/033_add_asset_event_tags_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/033_add_asset_event_tags_table.py @@ -5,6 +5,7 @@ Create Date: 2022-10-25 10:00:50.954192 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/035_add_run_job_index.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/035_add_run_job_index.py index c78fd60ba0a1..4f197b602f99 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/035_add_run_job_index.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/035_add_run_job_index.py @@ -5,6 +5,7 @@ Create Date: 2023-02-01 11:27:14.146322 """ + from dagster._core.storage.migration.utils import add_run_job_index, drop_run_job_index # revision identifiers, used by Alembic. diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/036_add_dynamic_partitions_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/036_add_dynamic_partitions_table.py index c0fc4d77c8fd..1e9737ccac60 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/036_add_dynamic_partitions_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/036_add_dynamic_partitions_table.py @@ -5,6 +5,7 @@ Create Date: 2023-01-19 11:41:41.062228 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/037_d9092588866f_add_primary_key_cols.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/037_d9092588866f_add_primary_key_cols.py index 254ae53e74bc..367f0fa3064c 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/037_d9092588866f_add_primary_key_cols.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/037_d9092588866f_add_primary_key_cols.py @@ -5,6 +5,7 @@ Create Date: 2023-03-03 14:20:07.082211 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import get_primary_key, has_column, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/038_701913684cb4_add_postgres_pks.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/038_701913684cb4_add_postgres_pks.py index a6f4ea35b2d0..3335d4cab34a 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/038_701913684cb4_add_postgres_pks.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/038_701913684cb4_add_postgres_pks.py @@ -5,6 +5,7 @@ Create Date: 2023-05-04 09:12:34.974039 """ + from alembic import op from dagster._core.storage.migration.utils import get_primary_key, has_column, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/040_add_in_progress_step_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/040_add_in_progress_step_table.py index ce70588a24bc..e3db1cabfab1 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/040_add_in_progress_step_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/040_add_in_progress_step_table.py @@ -5,6 +5,7 @@ Create Date: 2023-04-10 19:26:23.232433 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/041_add_asset_check_executions_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/041_add_asset_check_executions_table.py index facc436935c5..c42744ee6bdb 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/041_add_asset_check_executions_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/041_add_asset_check_executions_table.py @@ -5,6 +5,7 @@ Create Date: 2023-08-21 12:05:47.817280 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_index, has_table diff --git a/python_modules/dagster/dagster/_core/storage/alembic/versions/042_46b412388816_add_concurrency_limits_table.py b/python_modules/dagster/dagster/_core/storage/alembic/versions/042_46b412388816_add_concurrency_limits_table.py index 26cf59b230ad..c31de6a7c1f4 100644 --- a/python_modules/dagster/dagster/_core/storage/alembic/versions/042_46b412388816_add_concurrency_limits_table.py +++ b/python_modules/dagster/dagster/_core/storage/alembic/versions/042_46b412388816_add_concurrency_limits_table.py @@ -5,6 +5,7 @@ Create Date: 2023-12-01 14:35:47.622154 """ + import sqlalchemy as db from alembic import op from dagster._core.storage.migration.utils import has_table diff --git a/python_modules/dagster/dagster/_core/storage/dagster_run.py b/python_modules/dagster/dagster/_core/storage/dagster_run.py index 968fed8d8031..c3b8b2a0fba9 100644 --- a/python_modules/dagster/dagster/_core/storage/dagster_run.py +++ b/python_modules/dagster/dagster/_core/storage/dagster_run.py @@ -415,9 +415,9 @@ def tags_for_storage(self) -> Mapping[str, str]: if self.external_job_origin: # tag the run with a label containing the repository name / location name, to allow for # per-repository filtering of runs from the Dagster UI. - repository_tags[ - REPOSITORY_LABEL_TAG - ] = self.external_job_origin.external_repository_origin.get_label() + repository_tags[REPOSITORY_LABEL_TAG] = ( + self.external_job_origin.external_repository_origin.get_label() + ) if not self.tags: return repository_tags diff --git a/python_modules/dagster/dagster/_core/storage/db_io_manager.py b/python_modules/dagster/dagster/_core/storage/db_io_manager.py index 175e8d5644a5..2d9f1ad97189 100644 --- a/python_modules/dagster/dagster/_core/storage/db_io_manager.py +++ b/python_modules/dagster/dagster/_core/storage/db_io_manager.py @@ -68,28 +68,26 @@ def supported_types(self) -> Sequence[Type[object]]: class DbClient(Generic[T]): @staticmethod @abstractmethod - def delete_table_slice(context: OutputContext, table_slice: TableSlice, connection: T) -> None: - ... + def delete_table_slice( + context: OutputContext, table_slice: TableSlice, connection: T + ) -> None: ... @staticmethod @abstractmethod - def get_select_statement(table_slice: TableSlice) -> str: - ... + def get_select_statement(table_slice: TableSlice) -> str: ... @staticmethod @abstractmethod def ensure_schema_exists( context: OutputContext, table_slice: TableSlice, connection: T - ) -> None: - ... + ) -> None: ... @staticmethod @abstractmethod @contextmanager def connect( context: Union[OutputContext, InputContext], table_slice: TableSlice - ) -> Iterator[T]: - ... + ) -> Iterator[T]: ... class DbIOManager(IOManager): diff --git a/python_modules/dagster/dagster/_core/storage/input_manager.py b/python_modules/dagster/dagster/_core/storage/input_manager.py index a513ee33b127..c7ce37d98846 100644 --- a/python_modules/dagster/dagster/_core/storage/input_manager.py +++ b/python_modules/dagster/dagster/_core/storage/input_manager.py @@ -100,8 +100,7 @@ def copy_for_configured( @overload def input_manager( config_schema: InputLoadFn, -) -> InputManagerDefinition: - ... +) -> InputManagerDefinition: ... @overload @@ -111,8 +110,7 @@ def input_manager( input_config_schema: Optional[CoercableToConfigSchema] = None, required_resource_keys: Optional[AbstractSet[str]] = None, version: Optional[str] = None, -) -> Callable[[InputLoadFn], InputManagerDefinition]: - ... +) -> Callable[[InputLoadFn], InputManagerDefinition]: ... def input_manager( diff --git a/python_modules/dagster/dagster/_core/storage/io_manager.py b/python_modules/dagster/dagster/_core/storage/io_manager.py index df642cbb4525..ac675be2445c 100644 --- a/python_modules/dagster/dagster/_core/storage/io_manager.py +++ b/python_modules/dagster/dagster/_core/storage/io_manager.py @@ -156,8 +156,7 @@ def handle_output(self, context: "OutputContext", obj: Any) -> None: @overload -def io_manager(config_schema: IOManagerFunction) -> IOManagerDefinition: - ... +def io_manager(config_schema: IOManagerFunction) -> IOManagerDefinition: ... @overload @@ -168,8 +167,7 @@ def io_manager( input_config_schema: CoercableToConfigSchema = None, required_resource_keys: Optional[Set[str]] = None, version: Optional[str] = None, -) -> Callable[[IOManagerFunction], IOManagerDefinition]: - ... +) -> Callable[[IOManagerFunction], IOManagerDefinition]: ... def io_manager( diff --git a/python_modules/dagster/dagster/_core/storage/runs/base.py b/python_modules/dagster/dagster/_core/storage/runs/base.py index 9b1383956033..a73af65bf79d 100644 --- a/python_modules/dagster/dagster/_core/storage/runs/base.py +++ b/python_modules/dagster/dagster/_core/storage/runs/base.py @@ -383,5 +383,4 @@ def alembic_version(self) -> Optional[AlembicVersion]: return None @abstractmethod - def replace_job_origin(self, run: "DagsterRun", job_origin: "ExternalJobOrigin") -> None: - ... + def replace_job_origin(self, run: "DagsterRun", job_origin: "ExternalJobOrigin") -> None: ... diff --git a/python_modules/dagster/dagster/_core/system_config/objects.py b/python_modules/dagster/dagster/_core/system_config/objects.py index cdef748af182..d27a6f17caf4 100644 --- a/python_modules/dagster/dagster/_core/system_config/objects.py +++ b/python_modules/dagster/dagster/_core/system_config/objects.py @@ -1,4 +1,5 @@ """System-provided config objects and constructors.""" + from typing import ( AbstractSet, Any, diff --git a/python_modules/dagster/dagster/_core/telemetry.py b/python_modules/dagster/dagster/_core/telemetry.py index 80b00ca377af..8b9c9f2d9aef 100644 --- a/python_modules/dagster/dagster/_core/telemetry.py +++ b/python_modules/dagster/dagster/_core/telemetry.py @@ -107,16 +107,14 @@ @overload -def telemetry_wrapper(target_fn: T_Callable) -> T_Callable: - ... +def telemetry_wrapper(target_fn: T_Callable) -> T_Callable: ... @overload def telemetry_wrapper( *, metadata: Optional[Mapping[str, str]], -) -> Callable[[Callable[P, T]], Callable[P, T]]: - ... +) -> Callable[[Callable[P, T]], Callable[P, T]]: ... def telemetry_wrapper( diff --git a/python_modules/dagster/dagster/_core/types/decorator.py b/python_modules/dagster/dagster/_core/types/decorator.py index 342a91245072..731103faed33 100644 --- a/python_modules/dagster/dagster/_core/types/decorator.py +++ b/python_modules/dagster/dagster/_core/types/decorator.py @@ -15,15 +15,13 @@ def usable_as_dagster_type( name: Optional[str] = ..., description: Optional[str] = ..., loader: Optional["DagsterTypeLoader"] = ..., -) -> Callable[[T_Type], T_Type]: - ... +) -> Callable[[T_Type], T_Type]: ... @overload def usable_as_dagster_type( name: T_Type, -) -> T_Type: - ... +) -> T_Type: ... def usable_as_dagster_type( diff --git a/python_modules/dagster/dagster/_daemon/daemon.py b/python_modules/dagster/dagster/_daemon/daemon.py index f6980d1c0732..31695adac5d1 100644 --- a/python_modules/dagster/dagster/_daemon/daemon.py +++ b/python_modules/dagster/dagster/_daemon/daemon.py @@ -233,8 +233,7 @@ def core_loop( yield None @abstractmethod - def run_iteration(self, workspace_process_context: TContext) -> DaemonIterator: - ... + def run_iteration(self, workspace_process_context: TContext) -> DaemonIterator: ... class SchedulerDaemon(DagsterDaemon): diff --git a/python_modules/dagster/dagster/_grpc/compile.py b/python_modules/dagster/dagster/_grpc/compile.py index b1feeb583c6d..1e3afd5796f0 100644 --- a/python_modules/dagster/dagster/_grpc/compile.py +++ b/python_modules/dagster/dagster/_grpc/compile.py @@ -4,6 +4,7 @@ python -m dagster._grpc.compile """ + import os import shutil import subprocess diff --git a/python_modules/dagster/dagster/_serdes/config_class.py b/python_modules/dagster/dagster/_serdes/config_class.py index a82909e55d37..ea80661fc2c6 100644 --- a/python_modules/dagster/dagster/_serdes/config_class.py +++ b/python_modules/dagster/dagster/_serdes/config_class.py @@ -81,12 +81,10 @@ def info_dict(self) -> Mapping[str, Any]: } @overload - def rehydrate(self, as_type: None = ...) -> "ConfigurableClass": - ... + def rehydrate(self, as_type: None = ...) -> "ConfigurableClass": ... @overload - def rehydrate(self, as_type: Type[T_ConfigurableClass]) -> T_ConfigurableClass: - ... + def rehydrate(self, as_type: Type[T_ConfigurableClass]) -> T_ConfigurableClass: ... def rehydrate( self, as_type: Optional[Type[T_ConfigurableClass]] = None diff --git a/python_modules/dagster/dagster/_serdes/serdes.py b/python_modules/dagster/dagster/_serdes/serdes.py index a47ad0c75429..e94096729a3d 100644 --- a/python_modules/dagster/dagster/_serdes/serdes.py +++ b/python_modules/dagster/dagster/_serdes/serdes.py @@ -12,6 +12,7 @@ * This isn't meant to replace pickle in the conditions that pickle is reasonable to use (in memory, not human readable, etc) just handle the json case effectively. """ + import collections.abc import dataclasses from abc import ABC, abstractmethod @@ -236,8 +237,7 @@ def create() -> "WhitelistMap": @overload -def whitelist_for_serdes(__cls: T_Type) -> T_Type: - ... +def whitelist_for_serdes(__cls: T_Type) -> T_Type: ... @overload @@ -252,8 +252,7 @@ def whitelist_for_serdes( skip_when_empty_fields: Optional[AbstractSet[str]] = ..., field_serializers: Optional[Mapping[str, Type["FieldSerializer"]]] = None, is_pickleable: bool = True, -) -> Callable[[T_Type], T_Type]: - ... +) -> Callable[[T_Type], T_Type]: ... def whitelist_for_serdes( @@ -536,8 +535,7 @@ def __init__( self.field_serializers = field_serializers or {} @abstractmethod - def object_as_mapping(self, value: T) -> Mapping[str, PackableValue]: - ... + def object_as_mapping(self, value: T) -> Mapping[str, PackableValue]: ... def unpack( self, @@ -638,8 +636,7 @@ def after_pack(self, **packed_dict: JsonSerializableValue) -> Dict[str, JsonSeri @property @abstractmethod - def constructor_param_names(self) -> Sequence[str]: - ... + def constructor_param_names(self) -> Sequence[str]: ... def get_storage_name(self) -> str: return self.storage_name or self.klass.__name__ @@ -700,8 +697,7 @@ def unpack( __unpacked_value: UnpackedValue, whitelist_map: WhitelistMap, context: UnpackContext, - ) -> PackableValue: - ... + ) -> PackableValue: ... @abstractmethod def pack( @@ -709,8 +705,7 @@ def pack( __unpacked_value: Any, whitelist_map: WhitelistMap, descent_path: str, - ) -> JsonSerializableValue: - ... + ) -> JsonSerializableValue: ... class SetToSequenceFieldSerializer(FieldSerializer): @@ -752,8 +747,7 @@ def pack_value( val: T_Scalar, whitelist_map: WhitelistMap = ..., descent_path: Optional[str] = ..., -) -> T_Scalar: - ... +) -> T_Scalar: ... @overload # for all the types that serialize to JSON object @@ -769,8 +763,7 @@ def pack_value( ], whitelist_map: WhitelistMap = ..., descent_path: Optional[str] = ..., -) -> Mapping[str, JsonSerializableValue]: - ... +) -> Mapping[str, JsonSerializableValue]: ... @overload @@ -778,8 +771,7 @@ def pack_value( val: Sequence[PackableValue], whitelist_map: WhitelistMap = ..., descent_path: Optional[str] = ..., -) -> Sequence[JsonSerializableValue]: - ... +) -> Sequence[JsonSerializableValue]: ... def pack_value( @@ -899,8 +891,7 @@ def deserialize_value( val: str, as_type: Tuple[Type[T_PackableValue], Type[U_PackableValue]], whitelist_map: WhitelistMap = ..., -) -> Union[T_PackableValue, U_PackableValue]: - ... +) -> Union[T_PackableValue, U_PackableValue]: ... @overload @@ -908,8 +899,7 @@ def deserialize_value( val: str, as_type: Type[T_PackableValue], whitelist_map: WhitelistMap = ..., -) -> T_PackableValue: - ... +) -> T_PackableValue: ... @overload @@ -917,8 +907,7 @@ def deserialize_value( val: str, as_type: None = ..., whitelist_map: WhitelistMap = ..., -) -> PackableValue: - ... +) -> PackableValue: ... def deserialize_value( @@ -1016,8 +1005,7 @@ def unpack_value( as_type: Tuple[Type[T_PackableValue], Type[U_PackableValue]], whitelist_map: WhitelistMap = ..., context: Optional[UnpackContext] = ..., -) -> Union[T_PackableValue, U_PackableValue]: - ... +) -> Union[T_PackableValue, U_PackableValue]: ... @overload @@ -1026,8 +1014,7 @@ def unpack_value( as_type: Type[T_PackableValue], whitelist_map: WhitelistMap = ..., context: Optional[UnpackContext] = ..., -) -> T_PackableValue: - ... +) -> T_PackableValue: ... @overload @@ -1036,8 +1023,7 @@ def unpack_value( as_type: None = ..., whitelist_map: WhitelistMap = ..., context: Optional[UnpackContext] = ..., -) -> PackableValue: - ... +) -> PackableValue: ... def unpack_value( diff --git a/python_modules/dagster/dagster/_seven/__init__.py b/python_modules/dagster/dagster/_seven/__init__.py index ca5a90f03ad4..eae21cb9c510 100644 --- a/python_modules/dagster/dagster/_seven/__init__.py +++ b/python_modules/dagster/dagster/_seven/__init__.py @@ -1,4 +1,5 @@ """Internal py2/3 compatibility library. A little more than six.""" + import inspect import os import shlex diff --git a/python_modules/dagster/dagster/_utils/__init__.py b/python_modules/dagster/dagster/_utils/__init__.py index 022c4d7ddab8..ed8a34b78e08 100644 --- a/python_modules/dagster/dagster/_utils/__init__.py +++ b/python_modules/dagster/dagster/_utils/__init__.py @@ -270,18 +270,15 @@ def __hash__(self): @overload -def make_hashable(value: Union[List[Any], Set[Any]]) -> Tuple[Any, ...]: - ... +def make_hashable(value: Union[List[Any], Set[Any]]) -> Tuple[Any, ...]: ... @overload -def make_hashable(value: Dict[Any, Any]) -> Tuple[Tuple[Any, Any]]: - ... +def make_hashable(value: Dict[Any, Any]) -> Tuple[Tuple[Any, Any]]: ... @overload -def make_hashable(value: Any) -> Any: - ... +def make_hashable(value: Any) -> Any: ... def make_hashable(value: Any) -> Any: @@ -719,8 +716,7 @@ def normalize_to_repository( definitions_or_repository: Optional[Union["Definitions", "RepositoryDefinition"]] = ..., repository: Optional["RepositoryDefinition"] = ..., error_on_none: Literal[True] = ..., -) -> "RepositoryDefinition": - ... +) -> "RepositoryDefinition": ... @overload @@ -728,8 +724,7 @@ def normalize_to_repository( definitions_or_repository: Optional[Union["Definitions", "RepositoryDefinition"]] = ..., repository: Optional["RepositoryDefinition"] = ..., error_on_none: Literal[False] = ..., -) -> Optional["RepositoryDefinition"]: - ... +) -> Optional["RepositoryDefinition"]: ... def normalize_to_repository( diff --git a/python_modules/dagster/dagster/_utils/caching_instance_queryer.py b/python_modules/dagster/dagster/_utils/caching_instance_queryer.py index a0176398eefa..450ee50036c9 100644 --- a/python_modules/dagster/dagster/_utils/caching_instance_queryer.py +++ b/python_modules/dagster/dagster/_utils/caching_instance_queryer.py @@ -501,10 +501,10 @@ def get_materialized_partitions( before_cursor not in self._asset_partitions_cache or asset_key not in self._asset_partitions_cache[before_cursor] ): - self._asset_partitions_cache[before_cursor][ - asset_key - ] = self.instance.get_materialized_partitions( - asset_key=asset_key, before_cursor=before_cursor + self._asset_partitions_cache[before_cursor][asset_key] = ( + self.instance.get_materialized_partitions( + asset_key=asset_key, before_cursor=before_cursor + ) ) return self._asset_partitions_cache[before_cursor][asset_key] @@ -516,9 +516,9 @@ def get_materialized_partitions( def get_dynamic_partitions(self, partitions_def_name: str) -> Sequence[str]: """Returns a list of partitions for a partitions definition.""" if partitions_def_name not in self._dynamic_partitions_cache: - self._dynamic_partitions_cache[ - partitions_def_name - ] = self.instance.get_dynamic_partitions(partitions_def_name) + self._dynamic_partitions_cache[partitions_def_name] = ( + self.instance.get_dynamic_partitions(partitions_def_name) + ) return self._dynamic_partitions_cache[partitions_def_name] def has_dynamic_partition(self, partitions_def_name: str, partition_key: str) -> bool: @@ -749,9 +749,9 @@ def _asset_partition_versions_updated_after_cursor( } # keep track of the maximum storage id at which an asset partition was updated after for asset_partition in queryed_updated_asset_partitions: - self._asset_partition_versions_updated_after_cursor_cache[ - asset_partition - ] = after_cursor + self._asset_partition_versions_updated_after_cursor_cache[asset_partition] = ( + after_cursor + ) return {*updated_asset_partitions, *queryed_updated_asset_partitions} def get_asset_partitions_updated_after_cursor( diff --git a/python_modules/dagster/dagster/_utils/hosted_user_process.py b/python_modules/dagster/dagster/_utils/hosted_user_process.py index b4b07d4ff32b..0d696e3337bd 100644 --- a/python_modules/dagster/dagster/_utils/hosted_user_process.py +++ b/python_modules/dagster/dagster/_utils/hosted_user_process.py @@ -8,7 +8,6 @@ to be the case. """ - import dagster._check as check from dagster._core.definitions.reconstruct import ReconstructableJob, ReconstructableRepository from dagster._core.host_representation import ExternalJob diff --git a/python_modules/dagster/dagster/_utils/test/data_versions.py b/python_modules/dagster/dagster/_utils/test/data_versions.py index 35c443201239..4b17e3ff52bc 100644 --- a/python_modules/dagster/dagster/_utils/test/data_versions.py +++ b/python_modules/dagster/dagster/_utils/test/data_versions.py @@ -127,8 +127,7 @@ def materialize_asset( partition_key: Optional[str] = ..., run_config: Optional[Union[RunConfig, Mapping[str, Any]]] = ..., tags: Optional[Mapping[str, str]] = ..., -) -> MaterializationTable: - ... +) -> MaterializationTable: ... @overload @@ -141,8 +140,7 @@ def materialize_asset( partition_key: Optional[str] = ..., run_config: Optional[Union[RunConfig, Mapping[str, Any]]] = ..., tags: Optional[Mapping[str, str]] = ..., -) -> AssetMaterialization: - ... +) -> AssetMaterialization: ... # Use only for AssetsDefinition with one asset diff --git a/python_modules/dagster/dagster_tests/api_tests/error_on_load_repo.py b/python_modules/dagster/dagster_tests/api_tests/error_on_load_repo.py index 1702aa7974eb..fcba5660b31e 100644 --- a/python_modules/dagster/dagster_tests/api_tests/error_on_load_repo.py +++ b/python_modules/dagster/dagster_tests/api_tests/error_on_load_repo.py @@ -1,3 +1,3 @@ -"""A target that errors on load. -""" +"""A target that errors on load.""" + raise ValueError("User did something bad") diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py b/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py index 779d4cff490a..7da002b73f4d 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/partition_mapping_tests/test_asset_partition_mappings.py @@ -394,8 +394,7 @@ def downstream(context, upstream): return upstream class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): assert context.asset_key.path == ["staging", "upstream"] diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py index d56994451c91..e9deab2624a2 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_graph.py @@ -70,20 +70,16 @@ def asset_graph_from_assets_fixture(request) -> Callable[[List[AssetsDefinition] def test_basics(asset_graph_from_assets): @asset(code_version="1") - def asset0(): - ... + def asset0(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def asset1(asset0): - ... + def asset1(asset0): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def asset2(asset0): - ... + def asset2(asset0): ... @asset(partitions_def=HourlyPartitionsDefinition(start_date="2022-01-01-00:00")) - def asset3(asset1, asset2): - ... + def asset3(asset1, asset2): ... assets = [asset0, asset1, asset2, asset3] asset_graph = asset_graph_from_assets(assets) @@ -105,12 +101,10 @@ def test_get_children_partitions_unpartitioned_parent_partitioned_child( asset_graph_from_assets, ) -> None: @asset - def parent(): - ... + def parent(): ... @asset(partitions_def=StaticPartitionsDefinition(["a", "b"])) - def child(parent): - ... + def child(parent): ... with instance_for_test() as instance: current_time = pendulum.now("UTC") @@ -140,12 +134,10 @@ def test_get_parent_partitions_unpartitioned_child_partitioned_parent( asset_graph_from_assets: Callable[..., AssetGraph], ): @asset(partitions_def=StaticPartitionsDefinition(["a", "b"])) - def parent(): - ... + def parent(): ... @asset - def child(parent): - ... + def child(parent): ... with instance_for_test() as instance: current_time = pendulum.now("UTC") @@ -173,12 +165,10 @@ def child(parent): def test_get_children_partitions_fan_out(asset_graph_from_assets: Callable[..., AssetGraph]): @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def parent(): - ... + def parent(): ... @asset(partitions_def=HourlyPartitionsDefinition(start_date="2022-01-01-00:00")) - def child(parent): - ... + def child(parent): ... asset_graph = asset_graph_from_assets([parent, child]) with instance_for_test() as instance: @@ -209,12 +199,10 @@ def child(parent): def test_get_parent_partitions_fan_in(asset_graph_from_assets: Callable[..., AssetGraph]) -> None: @asset(partitions_def=HourlyPartitionsDefinition(start_date="2022-01-01-00:00")) - def parent(): - ... + def parent(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def child(parent): - ... + def child(parent): ... asset_graph = asset_graph_from_assets([parent, child]) @@ -250,12 +238,10 @@ def test_get_parent_partitions_non_default_partition_mapping( asset_graph_from_assets: Callable[..., AssetGraph], ): @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def parent(): - ... + def parent(): ... @asset(ins={"parent": AssetIn(partition_mapping=LastPartitionMapping())}) - def child(parent): - ... + def child(parent): ... asset_graph = asset_graph_from_assets([parent, child]) @@ -318,8 +304,7 @@ def description(self) -> str: raise NotImplementedError() @asset(partitions_def=StaticPartitionsDefinition(["1", "2", "3"])) - def parent(): - ... + def parent(): ... with pytest.warns( DeprecationWarning, @@ -334,8 +319,7 @@ def parent(): partitions_def=StaticPartitionsDefinition(["1", "2", "3"]), ins={"parent": AssetIn(partition_mapping=TrailingWindowPartitionMapping())}, ) - def child(parent): - ... + def child(parent): ... internal_asset_graph = InternalAssetGraph.from_assets([parent, child]) external_asset_graph = to_external_asset_graph([parent, child]) @@ -360,8 +344,7 @@ def test_required_multi_asset_sets_non_subsettable_multi_asset( asset_graph_from_assets: Callable[..., AssetGraph], ): @multi_asset(outs={"a": AssetOut(), "b": AssetOut()}) - def non_subsettable_multi_asset(): - ... + def non_subsettable_multi_asset(): ... asset_graph = asset_graph_from_assets([non_subsettable_multi_asset]) for asset_key in non_subsettable_multi_asset.keys: @@ -377,8 +360,7 @@ def test_required_multi_asset_sets_subsettable_multi_asset( outs={"a": AssetOut(), "b": AssetOut()}, can_subset=True, ) - def subsettable_multi_asset(): - ... + def subsettable_multi_asset(): ... asset_graph = asset_graph_from_assets([subsettable_multi_asset]) for asset_key in subsettable_multi_asset.keys: @@ -412,8 +394,7 @@ def test_required_multi_asset_sets_same_op_in_different_assets( asset_graph_from_assets: Callable[..., AssetGraph], ): @op - def op1(): - ... + def op1(): ... asset1 = AssetsDefinition.from_op(op1, keys_by_output_name={"result": AssetKey("a")}) asset2 = AssetsDefinition.from_op(op1, keys_by_output_name={"result": AssetKey("b")}) @@ -464,20 +445,16 @@ def test_bfs_filter_asset_subsets(asset_graph_from_assets: Callable[..., AssetGr daily_partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") @asset(partitions_def=daily_partitions_def) - def asset0(): - ... + def asset0(): ... @asset(partitions_def=daily_partitions_def) - def asset1(asset0): - ... + def asset1(asset0): ... @asset(partitions_def=daily_partitions_def) - def asset2(asset0): - ... + def asset2(asset0): ... @asset(partitions_def=HourlyPartitionsDefinition(start_date="2022-01-01-00:00")) - def asset3(asset1, asset2): - ... + def asset3(asset1, asset2): ... asset_graph = asset_graph_from_assets([asset0, asset1, asset2, asset3]) @@ -559,12 +536,10 @@ def test_bfs_filter_asset_subsets_different_mappings( daily_partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") @asset(partitions_def=daily_partitions_def) - def asset0(): - ... + def asset0(): ... @asset(partitions_def=daily_partitions_def) - def asset1(asset0): - ... + def asset1(asset0): ... @asset( partitions_def=daily_partitions_def, @@ -574,12 +549,10 @@ def asset1(asset0): ) }, ) - def asset2(asset0): - ... + def asset2(asset0): ... @asset(partitions_def=daily_partitions_def) - def asset3(asset1, asset2): - ... + def asset3(asset1, asset2): ... asset_graph = asset_graph_from_assets([asset0, asset1, asset2, asset3]) @@ -619,20 +592,16 @@ def test_asset_graph_subset_contains(asset_graph_from_assets: Callable[..., Asse daily_partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") @asset(partitions_def=daily_partitions_def) - def partitioned1(): - ... + def partitioned1(): ... @asset(partitions_def=daily_partitions_def) - def partitioned2(): - ... + def partitioned2(): ... @asset - def unpartitioned1(): - ... + def unpartitioned1(): ... @asset - def unpartitioned2(): - ... + def unpartitioned2(): ... asset_graph_subset = AssetGraphSubset( partitions_subsets_by_asset_key={ @@ -657,20 +626,16 @@ def test_asset_graph_difference(asset_graph_from_assets: Callable[..., AssetGrap daily_partitions_def = DailyPartitionsDefinition(start_date="2022-01-01") @asset(partitions_def=daily_partitions_def) - def partitioned1(): - ... + def partitioned1(): ... @asset(partitions_def=daily_partitions_def) - def partitioned2(): - ... + def partitioned2(): ... @asset - def unpartitioned1(): - ... + def unpartitioned1(): ... @asset - def unpartitioned2(): - ... + def unpartitioned2(): ... subset1 = AssetGraphSubset( partitions_subsets_by_asset_key={ @@ -722,45 +687,36 @@ def test_asset_graph_partial_deserialization(asset_graph_from_assets: Callable[. def get_ag1() -> AssetGraph: @asset(partitions_def=daily_partitions_def) - def partitioned1(): - ... + def partitioned1(): ... @asset(partitions_def=daily_partitions_def) - def partitioned2(): - ... + def partitioned2(): ... @asset - def unpartitioned1(): - ... + def unpartitioned1(): ... @asset - def unpartitioned2(): - ... + def unpartitioned2(): ... return asset_graph_from_assets([partitioned1, partitioned2, unpartitioned1, unpartitioned2]) def get_ag2() -> AssetGraph: @asset(partitions_def=daily_partitions_def) - def partitioned1(): - ... + def partitioned1(): ... # new partitions_def @asset(partitions_def=static_partitions_def) - def partitioned2(): - ... + def partitioned2(): ... # new asset @asset(partitions_def=static_partitions_def) - def partitioned3(): - ... + def partitioned3(): ... @asset - def unpartitioned2(): - ... + def unpartitioned2(): ... @asset - def unpartitioned3(): - ... + def unpartitioned3(): ... return asset_graph_from_assets( [partitioned1, partitioned2, partitioned3, unpartitioned2, unpartitioned3] @@ -806,12 +762,10 @@ def test_required_assets_and_checks_by_key_check_decorator( asset_graph_from_assets: Callable[..., AssetGraph], ): @asset - def asset0(): - ... + def asset0(): ... @asset_check(asset=asset0) - def check0(): - ... + def check0(): ... asset_graph = asset_graph_from_assets([asset0], asset_checks=[check0]) assert asset_graph.get_required_asset_and_check_keys(asset0.key) == set() @@ -825,12 +779,10 @@ def test_required_assets_and_checks_by_key_asset_decorator( bar_check = AssetCheckSpec(name="bar", asset="asset0") @asset(check_specs=[foo_check, bar_check]) - def asset0(): - ... + def asset0(): ... @asset_check(asset=asset0) - def check0(): - ... + def check0(): ... asset_graph = asset_graph_from_assets([asset0], asset_checks=[check0]) @@ -851,8 +803,7 @@ def test_required_assets_and_checks_by_key_multi_asset( outs={"asset0": AssetOut(), "asset1": AssetOut()}, check_specs=[foo_check, bar_check], ) - def asset_fn(): - ... + def asset_fn(): ... biz_check = AssetCheckSpec(name="bar", asset="subsettable_asset0") @@ -861,8 +812,7 @@ def asset_fn(): check_specs=[biz_check], can_subset=True, ) - def subsettable_asset_fn(): - ... + def subsettable_asset_fn(): ... asset_graph = asset_graph_from_assets([asset_fn, subsettable_asset_fn]) @@ -894,8 +844,7 @@ def test_required_assets_and_checks_by_key_multi_asset_single_asset( check_specs=[foo_check, bar_check], can_subset=True, ) - def asset_fn(): - ... + def asset_fn(): ... asset_graph = asset_graph_from_assets([asset_fn]) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_selection.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_selection.py index 3a3ed87be1e3..eb4d269d9491 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_selection.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_asset_selection.py @@ -369,8 +369,7 @@ def test_self_dep(partitions_def, partition_mapping): partitions_def=partitions_def, ins={"a": AssetIn(partition_mapping=partition_mapping)}, ) - def a(a): - ... + def a(a): ... assert AssetSelection.keys("a").resolve([a]) == {a.key} assert AssetSelection.keys("a").upstream().resolve([a]) == {a.key} @@ -381,12 +380,10 @@ def a(a): def test_from_coercible_multi_asset(): @multi_asset(outs={"asset1": AssetOut(), "asset2": AssetOut()}) - def my_multi_asset(): - ... + def my_multi_asset(): ... @asset - def other_asset(): - ... + def other_asset(): ... assert ( AssetSelection.from_coercible([my_multi_asset]).resolve([my_multi_asset, other_asset]) @@ -396,12 +393,10 @@ def other_asset(): def test_from_coercible_tuple(): @asset - def foo(): - ... + def foo(): ... @asset - def bar(): - ... + def bar(): ... assert AssetSelection.from_coercible((foo, bar)).resolve([foo, bar]) == { AssetKey("foo"), @@ -542,16 +537,13 @@ def resolve_inner(self, asset_graph: AssetGraph) -> AbstractSet[AssetKey]: return asset_graph.materializable_asset_keys - {AssetKey("asset2")} @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset_check(asset=asset1) - def check1(): - ... + def check1(): ... asset_graph = InternalAssetGraph.from_assets([asset1, asset2], asset_checks=[check1]) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py index 24f20e4b6313..bbe2bce2827a 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets.py @@ -150,8 +150,7 @@ def test_with_replaced_description() -> None: "baz": AssetOut(description="baz"), } ) - def abc(): - ... + def abc(): ... assert abc.descriptions_by_key == { AssetKey("foo"): "foo", @@ -176,12 +175,10 @@ def abc(): } @op - def op1(): - ... + def op1(): ... @op - def op2(): - ... + def op2(): ... @graph_multi_asset( outs={"foo": AssetOut(description="foo"), "bar": AssetOut(description="bar")} @@ -215,8 +212,7 @@ def test_with_replaced_metadata() -> None: "baz": AssetOut(metadata={"baz": "baz"}), } ) - def abc(): - ... + def abc(): ... assert abc.metadata_by_key == { AssetKey("foo"): {"foo": "foo"}, @@ -488,8 +484,7 @@ def baz(): def test_to_source_assets(): @asset(metadata={"a": "b"}, io_manager_key="abc", description="blablabla") - def my_asset(): - ... + def my_asset(): ... assert ( my_asset.to_source_assets() @@ -1541,8 +1536,7 @@ def a(a): del a class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): assert context.asset_key.path[-1] == "a" @@ -1887,8 +1881,7 @@ def table_A(): table_c_no_dep = AssetSpec("table_C") @multi_asset(specs=[table_b, table_c]) - def deps_in_specs(): - ... + def deps_in_specs(): ... result = materialize([deps_in_specs, table_A]) assert result.success @@ -1900,12 +1893,10 @@ def deps_in_specs(): with pytest.raises(DagsterInvalidDefinitionError, match="Can not pass deps and specs"): @multi_asset(specs=[table_b_no_dep, table_c_no_dep], deps=["table_A"]) - def no_deps_in_specs(): - ... + def no_deps_in_specs(): ... @multi_asset(specs=[table_b, table_c]) - def also_input(table_A): - ... + def also_input(table_A): ... result = materialize([also_input, table_A]) assert result.success @@ -1920,8 +1911,7 @@ def also_input(table_A): ): @multi_asset(specs=[table_b, table_c]) - def rogue_input(table_X): - ... + def rogue_input(table_X): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1929,8 +1919,7 @@ def rogue_input(table_X): ): @multi_asset(specs=[table_b_no_dep, table_c_no_dep]) - def no_spec_deps_but_input(table_A): - ... + def no_spec_deps_but_input(table_A): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1941,8 +1930,7 @@ def no_spec_deps_but_input(table_A): specs=[table_b_no_dep, table_c_no_dep], deps=[table_A], ) - def use_deps(): - ... + def use_deps(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1953,8 +1941,7 @@ def use_deps(): specs=[table_b_no_dep, table_c_no_dep], internal_asset_deps={"table_C": {AssetKey("table_B")}}, ) - def use_internal_deps(): - ... + def use_internal_deps(): ... def test_asset_key_on_context(): diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py index 9aca8718eec4..74588e8d18d4 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_assets_job.py @@ -1877,8 +1877,7 @@ def my_derived_asset(my_source_asset): def test_resolve_dependency_in_group(): @asset(key_prefix="abc") - def asset1(): - ... + def asset1(): ... @asset def asset2(context, asset1): @@ -1891,8 +1890,7 @@ def asset2(context, asset1): def test_resolve_dependency_fail_across_groups(): @asset(key_prefix="abc", group_name="other") - def asset1(): - ... + def asset1(): ... @asset def asset2(asset1): @@ -1911,8 +1909,7 @@ def asset2(asset1): def test_resolve_dependency_multi_asset_different_groups(): @asset(key_prefix="abc", group_name="a") - def upstream(): - ... + def upstream(): ... @op(out={"ns1": Out(), "ns2": Out()}) def op1(upstream): @@ -1938,24 +1935,19 @@ def op1(upstream): def test_get_base_asset_jobs_multiple_partitions_defs(): @asset(partitions_def=DailyPartitionsDefinition(start_date="2021-05-05")) - def daily_asset(): - ... + def daily_asset(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2021-05-05")) - def daily_asset2(): - ... + def daily_asset2(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2020-05-05")) - def daily_asset_different_start_date(): - ... + def daily_asset_different_start_date(): ... @asset(partitions_def=HourlyPartitionsDefinition(start_date="2021-05-05-00:00")) - def hourly_asset(): - ... + def hourly_asset(): ... @asset - def unpartitioned_asset(): - ... + def unpartitioned_asset(): ... jobs = get_base_asset_jobs( assets=[ @@ -1986,24 +1978,20 @@ def unpartitioned_asset(): @ignore_warning("Function `observable_source_asset` is experimental") def test_get_base_asset_jobs_multiple_partitions_defs_and_observable_assets(): - class B: - ... + class B: ... partitions_a = StaticPartitionsDefinition(["a1"]) @observable_source_asset(partitions_def=partitions_a) - def asset_a(): - ... + def asset_a(): ... partitions_b = StaticPartitionsDefinition(["b1"]) @observable_source_asset(partitions_def=partitions_b) - def asset_b(): - ... + def asset_b(): ... @asset(partitions_def=partitions_b) - def asset_x(asset_b: B): - ... + def asset_x(asset_b: B): ... jobs = get_base_asset_jobs( assets=[ @@ -2086,8 +2074,7 @@ def test_selection_multi_component(): source_asset = SourceAsset(["apple", "banana"]) @asset(key_prefix="abc") - def asset1(): - ... + def asset1(): ... assert Definitions( assets=[source_asset, asset1], jobs=[define_asset_job("something", selection="abc/asset1")] diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py index fb44c4bebffa..6f906be9e4ed 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_decorators.py @@ -576,8 +576,7 @@ def test_op_tags(): tags_stringified = {"apple": "banana", "orange": '{"rind": "fsd", "segment": "fjdskl"}'} @asset(op_tags=tags) - def my_asset(): - ... + def my_asset(): ... assert my_asset.op.tags == tags_stringified @@ -611,8 +610,7 @@ def my_asset(context, **kwargs): assert my_asset.op(build_op_context(), upstream=5) == 7 @asset - def upstream(): - ... + def upstream(): ... assert materialize_to_memory([upstream, my_asset]).success @@ -631,8 +629,7 @@ def my_asset(**kwargs): assert my_asset.op(upstream=5) == (7,) @asset - def upstream(): - ... + def upstream(): ... assert materialize_to_memory([upstream, my_asset]).success @@ -652,8 +649,7 @@ def my_asset(context, **kwargs): assert my_asset.op(build_op_context(), upstream=5) == (7,) @asset - def upstream(): - ... + def upstream(): ... assert materialize_to_memory([upstream, my_asset]).success @@ -733,8 +729,7 @@ def test_asset_retry_policy(): retry_policy = RetryPolicy() @asset(retry_policy=retry_policy) - def my_asset(): - ... + def my_asset(): ... assert my_asset.op.retry_policy == retry_policy @@ -749,8 +744,7 @@ def test_multi_asset_retry_policy(): }, retry_policy=retry_policy, ) - def my_asset(): - ... + def my_asset(): ... assert my_asset.op.retry_policy == retry_policy @@ -839,8 +833,7 @@ def b(b): def test_asset_in_nothing(): @asset(ins={"upstream": AssetIn(dagster_type=Nothing)}) - def asset1(): - ... + def asset1(): ... assert AssetKey("upstream") in asset1.keys_by_input_name.values() assert materialize_to_memory([asset1]).success @@ -848,8 +841,7 @@ def asset1(): def test_asset_in_nothing_and_something(): @asset - def other_upstream(): - ... + def other_upstream(): ... @asset(ins={"upstream": AssetIn(dagster_type=Nothing)}) def asset1(other_upstream): @@ -938,8 +930,7 @@ def test_graph_asset_partition_mapping(): partitions_def = StaticPartitionsDefinition(["a", "b", "c"]) @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @op(ins={"in1": In(Nothing)}) def my_op(context): @@ -1245,8 +1236,7 @@ def test_multi_asset_with_auto_materialize_policy(): "o3": AssetOut(auto_materialize_policy=AutoMaterializePolicy.lazy()), } ) - def my_asset(): - ... + def my_asset(): ... assert my_asset.auto_materialize_policies_by_key == { AssetKey("o2"): AutoMaterializePolicy.eager(), @@ -1281,8 +1271,7 @@ def test_error_on_asset_key_provided(): ): @asset(key="the_asset", key_prefix="foo") - def one(): - ... + def one(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1290,8 +1279,7 @@ def one(): ): @asset(key="the_asset", name="foo") - def two(): - ... + def two(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1299,8 +1287,7 @@ def two(): ): @asset(key="the_asset", name="foo", key_prefix="bar") - def three(): - ... + def three(): ... def test_dynamic_graph_asset_ins(): @@ -1313,8 +1300,7 @@ def wait_until_job_done(x): return x @asset - def foo(): - ... + def foo(): ... all_assets = [foo] @@ -1333,8 +1319,7 @@ def test_graph_inputs_error(): try: @graph_asset(ins={"start": AssetIn(dagster_type=Nothing)}) - def _(): - ... + def _(): ... except DagsterInvalidDefinitionError as err: assert "except for Ins that have the Nothing dagster_type" not in str(err) @@ -1342,8 +1327,7 @@ def _(): try: @graph(ins={"start": GraphIn()}) - def _(): - ... + def _(): ... except DagsterInvalidDefinitionError as err: assert "except for Ins that have the Nothing dagster_type" not in str(err) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_external_asset_graph.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_external_asset_graph.py index c8e5bbea4a6a..1fa967b0a0cd 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_external_asset_graph.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_external_asset_graph.py @@ -33,16 +33,14 @@ @asset -def asset1(): - ... +def asset1(): ... defs1 = Definitions(assets=[asset1]) @asset -def asset2(): - ... +def asset2(): ... defs2 = Definitions(assets=[asset2]) @@ -60,8 +58,7 @@ def downstream(asset1): @asset(deps=[asset1]) -def downstream_non_arg_dep(): - ... +def downstream_non_arg_dep(): ... downstream_defs_no_source = Definitions(assets=[downstream_non_arg_dep]) @@ -441,13 +438,11 @@ def test_cycle_status(instance): @asset -def single_materializable_asset(): - ... +def single_materializable_asset(): ... @observable_source_asset -def single_observable_asset(): - ... +def single_observable_asset(): ... dup_materialization_defs_a = Definitions(assets=[single_materializable_asset]) diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py index df9dda81ceca..a25d3579b1d9 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize.py @@ -358,12 +358,10 @@ def asset1(): def test_selection(): @asset - def upstream(): - ... + def upstream(): ... @asset - def downstream(upstream): - ... + def downstream(upstream): ... assets = [upstream, downstream] diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize_to_memory.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize_to_memory.py index 9a9b22d2c28f..31a787d9e671 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize_to_memory.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_materialize_to_memory.py @@ -326,12 +326,10 @@ def asset1(): def test_selection(): @asset - def upstream(): - ... + def upstream(): ... @asset - def downstream(upstream): - ... + def downstream(upstream): ... assets = [upstream, downstream] diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_resolved_asset_deps.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_resolved_asset_deps.py index dadb477d8334..25211b622cf9 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_resolved_asset_deps.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_resolved_asset_deps.py @@ -4,12 +4,10 @@ def test_same_name_twice_and_downstream(): @asset(name="apple", key_prefix="a") - def asset1(): - ... + def asset1(): ... @asset(name="apple", key_prefix="b") - def asset2(): - ... + def asset2(): ... @asset(ins={"apple": AssetIn(key_prefix="a")}) def asset3(apple): @@ -36,25 +34,21 @@ def multi_downstream(upstream): def test_input_has_asset_key(): @asset(key_prefix="a") - def asset1(): - ... + def asset1(): ... @asset(deps=[AssetKey(["b", "asset1"])]) - def asset2(): - ... + def asset2(): ... assert len(resolve_assets_def_deps([asset1, asset2], [])) == 0 def test_upstream_same_name_as_asset(): @asset(deps=[AssetKey("asset1")], key_prefix="b") - def asset1(): - ... + def asset1(): ... assert len(resolve_assets_def_deps([asset1], [])) == 0 @multi_asset(outs={"asset1": AssetOut(key_prefix="b")}, deps=[AssetKey(["a", "asset1"])]) - def multi_asset1(): - ... + def multi_asset1(): ... assert len(resolve_assets_def_deps([multi_asset1], [])) == 0 diff --git a/python_modules/dagster/dagster_tests/asset_defs_tests/test_unresolved_asset_job.py b/python_modules/dagster/dagster_tests/asset_defs_tests/test_unresolved_asset_job.py index 2bdfd3709057..d5b50193020c 100644 --- a/python_modules/dagster/dagster_tests/asset_defs_tests/test_unresolved_asset_job.py +++ b/python_modules/dagster/dagster_tests/asset_defs_tests/test_unresolved_asset_job.py @@ -437,16 +437,13 @@ def test_define_selection_job(job_selection, expected_assets, use_multi, prefixe def test_define_selection_job_assets_definition_selection(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset - def asset3(): - ... + def asset3(): ... all_assets = [asset1, asset2, asset3] diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py index 45e9b84b28a0..25c639c7d926 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/assets.py @@ -2,28 +2,23 @@ @asset -def asset1() -> None: - ... +def asset1() -> None: ... @asset(key_prefix=["some", "key", "prefix"]) -def asset_with_prefix() -> None: - ... +def asset_with_prefix() -> None: ... @asset(deps=[asset1]) -def downstream_asset() -> None: - ... +def downstream_asset() -> None: ... @asset(partitions_def=StaticPartitionsDefinition(["one", "two", "three"])) -def partitioned_asset() -> None: - ... +def partitioned_asset() -> None: ... @asset(partitions_def=StaticPartitionsDefinition(["apple", "banana", "pear"])) -def differently_partitioned_asset() -> None: - ... +def differently_partitioned_asset() -> None: ... @asset diff --git a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_telemetry.py b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_telemetry.py index 002a8ffab208..74a90df50a32 100644 --- a/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_telemetry.py +++ b/python_modules/dagster/dagster_tests/cli_tests/command_tests/test_telemetry.py @@ -211,16 +211,13 @@ def test_update_repo_stats_dynamic_partitions(caplog): def test_get_stats_from_external_repo_partitions(instance): @asset(partitions_def=StaticPartitionsDefinition(["foo", "bar"])) - def asset1(): - ... + def asset1(): ... @asset(partitions_def=DailyPartitionsDefinition(start_date="2022-01-01")) - def asset2(): - ... + def asset2(): ... @asset - def asset3(): - ... + def asset3(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -242,8 +239,7 @@ def test_get_stats_from_external_repo_multi_partitions(instance): } ) ) - def multi_partitioned_asset(): - ... + def multi_partitioned_asset(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -261,8 +257,7 @@ def test_get_stats_from_external_repo_source_assets(instance): source_asset1 = SourceAsset("source_asset1") @asset - def asset1(): - ... + def asset1(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -279,12 +274,10 @@ def test_get_stats_from_external_repo_observable_source_assets(instance): source_asset1 = SourceAsset("source_asset1") @observable_source_asset - def source_asset2(): - ... + def source_asset2(): ... @asset - def asset1(): - ... + def asset1(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -300,12 +293,10 @@ def asset1(): def test_get_stats_from_external_repo_freshness_policies(instance): @asset(freshness_policy=FreshnessPolicy(maximum_lag_minutes=30)) - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -320,16 +311,13 @@ def asset2(): def test_get_status_from_external_repo_auto_materialize_policy(instance): @asset(auto_materialize_policy=AutoMaterializePolicy.lazy()) - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset(auto_materialize_policy=AutoMaterializePolicy.eager()) - def asset3(): - ... + def asset3(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -345,12 +333,10 @@ def asset3(): def test_get_stats_from_external_repo_code_versions(instance): @asset(code_version="hello") - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -365,20 +351,16 @@ def asset2(): def test_get_stats_from_external_repo_code_checks(instance): @asset - def my_asset(): - ... + def my_asset(): ... @asset_check(asset=my_asset) - def my_check(): - ... + def my_check(): ... @asset_check(asset=my_asset) - def my_check_2(): - ... + def my_check_2(): ... @asset - def my_other_asset(): - ... + def my_other_asset(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -396,12 +378,10 @@ def my_other_asset(): def test_get_stats_from_external_repo_dbt(instance): @asset(compute_kind="dbt") - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -426,8 +406,7 @@ class CustomResource(ConfigurableResource): baz: str @asset - def asset1(my_resource: MyResource, custom_resource: CustomResource): - ... + def asset1(my_resource: MyResource, custom_resource: CustomResource): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -473,8 +452,7 @@ def load_input(self, context: InputContext) -> Any: return 1 @asset - def asset1(): - ... + def asset1(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -507,8 +485,7 @@ def custom_resource(): return 2 @asset(required_resource_keys={"my_resource", "custom_resource"}) - def asset1(): - ... + def asset1(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -541,8 +518,7 @@ def custom_io_manager(): return 2 @asset - def asset1(): - ... + def asset1(): ... external_repo = ExternalRepository( external_repository_data_from_def( @@ -615,12 +591,10 @@ def my_io_manager(): return 1 @asset - def asset1(my_resource: MyResource): - ... + def asset1(my_resource: MyResource): ... @asset(required_resource_keys={"my_other_resource"}) - def asset2(): - ... + def asset2(): ... external_repo = ExternalRepository( external_repository_data_from_def( diff --git a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/decorators_tests/test_sensor_decorator.py b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/decorators_tests/test_sensor_decorator.py index e4c606158308..795acba8d9c4 100644 --- a/python_modules/dagster/dagster_tests/core_tests/definitions_tests/decorators_tests/test_sensor_decorator.py +++ b/python_modules/dagster/dagster_tests/core_tests/definitions_tests/decorators_tests/test_sensor_decorator.py @@ -3,27 +3,22 @@ def test_coerce_to_asset_selection(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset - def asset3(): - ... + def asset3(): ... assets = [asset1, asset2, asset3] @sensor(asset_selection=["asset1", "asset2"]) - def sensor1(): - ... + def sensor1(): ... assert sensor1.asset_selection.resolve(assets) == {AssetKey("asset1"), AssetKey("asset2")} @sensor(asset_selection=[asset1, asset2]) - def sensor2(): - ... + def sensor2(): ... assert sensor2.asset_selection.resolve(assets) == {AssetKey("asset1"), AssetKey("asset2")} diff --git a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py index 999a78e16397..9abdc32988b7 100644 --- a/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py +++ b/python_modules/dagster/dagster_tests/core_tests/execution_tests/test_asset_backfill_with_backfill_policies.py @@ -500,8 +500,7 @@ def test_dynamic_partitions_multi_run_backfill_policy(): backfill_policy=BackfillPolicy.multi_run(), partitions_def=DynamicPartitionsDefinition(name="apple"), ) - def asset1() -> None: - ... + def asset1() -> None: ... assets_by_repo_name = {"repo": [asset1]} asset_graph = get_asset_graph(assets_by_repo_name) @@ -543,8 +542,7 @@ def test_dynamic_partitions_single_run_backfill_policy(): backfill_policy=BackfillPolicy.single_run(), partitions_def=DynamicPartitionsDefinition(name="apple"), ) - def asset1() -> None: - ... + def asset1() -> None: ... assets_by_repo_name = {"repo": [asset1]} asset_graph = get_asset_graph(assets_by_repo_name) diff --git a/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph.py b/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph.py index 21b993d8222e..80edbd750009 100644 --- a/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph.py +++ b/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph.py @@ -1238,8 +1238,7 @@ def mapped_out(): def test_infer_graph_input_type_from_inner_input(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @graph def graph1(in1): @@ -1266,8 +1265,7 @@ def graph1(in1): def test_infer_graph_input_type_from_inner_input_explicit_any(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @graph def graph1(in1: Any): @@ -1280,8 +1278,7 @@ def graph1(in1: Any): def test_infer_graph_input_type_from_inner_input_explicit_graphin_type(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @graph def graph1(in1: int): @@ -1292,12 +1289,10 @@ def graph1(in1: int): def test_infer_graph_input_type_from_multiple_inner_inputs(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @op(ins={"in2": In(Nothing)}) - def op2(): - ... + def op2(): ... @graph def graph1(in1): @@ -1311,8 +1306,7 @@ def graph1(in1): def test_dont_infer_graph_input_type_from_different_inner_inputs(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @op def op2(in2): @@ -1331,8 +1325,7 @@ def graph1(in1): def test_infer_graph_input_type_from_inner_inner_input(): @op(ins={"in1": In(Nothing)}) - def op1(): - ... + def op1(): ... @graph def inner(in1): diff --git a/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph_source_asset_input.py b/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph_source_asset_input.py index 96d06c50056d..1b96c956e8d2 100644 --- a/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph_source_asset_input.py +++ b/python_modules/dagster/dagster_tests/core_tests/graph_tests/test_graph_source_asset_input.py @@ -14,8 +14,7 @@ def make_io_manager(source_asset: SourceAsset, input_value=5, expected_metadata={}): class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): self.loaded_input = True @@ -68,8 +67,7 @@ def test_partitioned_source_asset_input_value(): asset1 = SourceAsset("asset1", partitions_def=partitions_def) class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): self.loaded_input = True @@ -99,8 +97,7 @@ def test_non_partitioned_job_partitioned_source_asset(): asset1 = SourceAsset("asset1", partitions_def=partitions_def) class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): self.loaded_input = True diff --git a/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_data.py b/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_data.py index ccf8dc718059..11f907be5bd3 100644 --- a/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_data.py +++ b/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_data.py @@ -1123,8 +1123,7 @@ def create_twenty(thirteen, six): def test_deps_resolve_group(): @asset(key_prefix="abc") - def asset1(): - ... + def asset1(): ... @asset def asset2(asset1): @@ -1256,8 +1255,7 @@ def test_external_multi_partitions_def(): def test_graph_asset_description(): @op - def op1(): - ... + def op1(): ... @graph_asset(description="bar") def foo(): @@ -1273,12 +1271,10 @@ def foo(): def test_graph_multi_asset_description(): @op - def op1(): - ... + def op1(): ... @op - def op2(): - ... + def op2(): ... @graph_multi_asset( outs={ diff --git a/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_sensor_data.py b/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_sensor_data.py index fa85a672093b..48c61211b27f 100644 --- a/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_sensor_data.py +++ b/python_modules/dagster/dagster_tests/core_tests/host_representation_tests/test_external_sensor_data.py @@ -7,8 +7,7 @@ def test_external_sensor_has_asset_selection(): @sensor(asset_selection=["asset1", "asset2"]) - def sensor1(): - ... + def sensor1(): ... defs = Definitions(sensors=[sensor1]) @@ -20,20 +19,17 @@ def sensor1(): def test_unserializable_asset_selection(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... class MySpecialAssetSelection(AssetSelection, frozen=True): def resolve_inner(self, asset_graph: AssetGraph) -> AbstractSet[AssetKey]: return asset_graph.materializable_asset_keys - {AssetKey("asset2")} @sensor(asset_selection=MySpecialAssetSelection()) - def sensor1(): - ... + def sensor1(): ... defs = Definitions(assets=[asset1, asset2]) assert external_sensor_data_from_def( diff --git a/python_modules/dagster/dagster_tests/core_tests/instance_tests/test_instance_data_versions.py b/python_modules/dagster/dagster_tests/core_tests/instance_tests/test_instance_data_versions.py index f020b7e18c54..d9b7361b2e2c 100644 --- a/python_modules/dagster/dagster_tests/core_tests/instance_tests/test_instance_data_versions.py +++ b/python_modules/dagster/dagster_tests/core_tests/instance_tests/test_instance_data_versions.py @@ -112,12 +112,10 @@ def test_compute_logical_data_version_unknown_dep_version(): def test_get_latest_materialization_code_versions(): @asset(code_version="abc") - def has_code_version(): - ... + def has_code_version(): ... @asset - def has_no_code_version(): - ... + def has_no_code_version(): ... instance = DagsterInstance.ephemeral() materialize([has_code_version, has_no_code_version], instance=instance) diff --git a/python_modules/dagster/dagster_tests/core_tests/pythonic_config_tests/test_configured.py b/python_modules/dagster/dagster_tests/core_tests/pythonic_config_tests/test_configured.py index 31773274513c..21b8cd103185 100644 --- a/python_modules/dagster/dagster_tests/core_tests/pythonic_config_tests/test_configured.py +++ b/python_modules/dagster/dagster_tests/core_tests/pythonic_config_tests/test_configured.py @@ -88,8 +88,7 @@ class DoSomethingSimplifiedConfig(Config): ): @configured(do_something, config_schema={"simplified_param": str}) - def do_something_simplified(config_in: DoSomethingSimplifiedConfig): - ... + def do_something_simplified(config_in: DoSomethingSimplifiedConfig): ... def test_config_annotation_extra_param_err() -> None: @@ -110,5 +109,4 @@ class DoSomethingSimplifiedConfig(Config): ): @configured(do_something) - def do_something_simplified(config_in: DoSomethingSimplifiedConfig, useless_param: str): - ... + def do_something_simplified(config_in: DoSomethingSimplifiedConfig, useless_param: str): ... diff --git a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_binding.py b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_binding.py index 4e43d1e38a01..560af760cf20 100644 --- a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_binding.py +++ b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_binding.py @@ -260,8 +260,7 @@ def hello_world_job(): hello_world_op() @sensor(job=hello_world_job) - def hello_world_sensor(): - ... + def hello_world_sensor(): ... hello_world_schedule = ScheduleDefinition( name="hello_world_schedule", cron_schedule="* * * * *", job=hello_world_job @@ -314,8 +313,7 @@ def hello_world_job(): hello_world_op() @sensor(job_name="hello_world_job") - def hello_world_sensor(): - ... + def hello_world_sensor(): ... hello_world_schedule = ScheduleDefinition( name="hello_world_schedule", cron_schedule="* * * * *", job_name="hello_world_job" diff --git a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_configured.py b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_configured.py index 8186eb6973c4..cc85e4740ac5 100644 --- a/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_configured.py +++ b/python_modules/dagster/dagster_tests/core_tests/resource_tests/pythonic_resources/test_configured.py @@ -111,8 +111,7 @@ class MyResourceSimplifiedConfig(Config): ): @configured(MyResource, config_schema={"simplified_param": str}) - def my_resource_simplified(config_in: MyResourceSimplifiedConfig): - ... + def my_resource_simplified(config_in: MyResourceSimplifiedConfig): ... def test_config_annotation_extra_param_err() -> None: @@ -129,8 +128,7 @@ class MyResourceSimplifiedConfig(Config): ): @configured(MyResource) - def my_resource_simplified(config_in: MyResourceSimplifiedConfig, useless_param: str): - ... + def my_resource_simplified(config_in: MyResourceSimplifiedConfig, useless_param: str): ... def test_factory_resource_pattern_noargs() -> None: diff --git a/python_modules/dagster/dagster_tests/core_tests/serdes_tests/subprocess_with_interrupt_support.py b/python_modules/dagster/dagster_tests/core_tests/serdes_tests/subprocess_with_interrupt_support.py index 3d18951091cd..4ddc057e0d68 100644 --- a/python_modules/dagster/dagster_tests/core_tests/serdes_tests/subprocess_with_interrupt_support.py +++ b/python_modules/dagster/dagster_tests/core_tests/serdes_tests/subprocess_with_interrupt_support.py @@ -1,4 +1,5 @@ """Test interrupt handling.""" + import sys import time diff --git a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_repository_snap.py b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_repository_snap.py index c37cdbb9a45d..7a17c45701ea 100644 --- a/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_repository_snap.py +++ b/python_modules/dagster/dagster_tests/core_tests/snap_tests/test_repository_snap.py @@ -610,12 +610,10 @@ def my_asset(): pass @asset_check(asset=my_asset) - def my_asset_check(): - ... + def my_asset_check(): ... @asset_check(asset=my_asset) - def my_asset_check_2(): - ... + def my_asset_check_2(): ... defs = Definitions( assets=[my_asset], @@ -641,8 +639,7 @@ def my_asset(): pass @asset_check(asset=my_asset) - def my_asset_check(): - ... + def my_asset_check(): ... defs = Definitions( assets=[my_asset], @@ -668,8 +665,7 @@ def my_asset(): pass @asset_check(asset=my_asset) - def my_asset_check(): - ... + def my_asset_check(): ... my_job = define_asset_job("my_job", [my_asset]) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py index 8d5cf5bb029a..3b134b198795 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/asset_daemon_scenario.py @@ -220,8 +220,7 @@ class AssetSpecWithPartitionsDef( AssetSpec._fields + ("partitions_def",), defaults=(None,) * (1 + len(AssetSpec._fields)), ) -): - ... +): ... class MultiAssetSpec(NamedTuple): diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/definition_change_scenarios.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/definition_change_scenarios.py index 4c8c50e29185..39d10335ae42 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/definition_change_scenarios.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/definition_change_scenarios.py @@ -1,5 +1,4 @@ -"""Scenarios where the set of asset definitions changes between ticks. -""" +"""Scenarios where the set of asset definitions changes between ticks.""" from ..base_scenario import AssetReconciliationScenario from .basic_scenarios import one_asset, two_assets_in_sequence diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/scenarios.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/scenarios.py index 4561ffab31a0..086f51527b92 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/scenarios.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/scenarios/scenarios.py @@ -40,9 +40,9 @@ for location_name, assets in scenario.code_locations.items(): d = Definitions(assets=assets, executor=in_process_executor) - globals()[ - "hacky_daemon_repo_" + scenario_name + "_" + location_name - ] = d.get_repository_def() + globals()["hacky_daemon_repo_" + scenario_name + "_" + location_name] = ( + d.get_repository_def() + ) else: assert scenario.code_locations is None diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_cursor.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_cursor.py index f4bc6a461069..40acb617eb6f 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_cursor.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_asset_daemon_cursor.py @@ -53,12 +53,10 @@ def test_asset_reconciliation_cursor_auto_observe_backcompat() -> None: partitions_def = StaticPartitionsDefinition(["a", "b", "c"]) @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... handled_root_partitions_by_asset_key = { asset1.key: partitions_def.subset_with_partition_keys(["a", "b"]) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_auto_observe.py b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_auto_observe.py index 6b5b4b91e2d2..84b6a0726883 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_auto_observe.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/auto_materialize_tests/test_auto_observe.py @@ -13,8 +13,7 @@ def test_single_observable_source_asset_no_auto_observe(): @observable_source_asset - def asset1(): - ... + def asset1(): ... asset_graph = InternalAssetGraph.from_assets([asset1]) @@ -48,8 +47,7 @@ def asset1(): @fixture(params=[True, False], ids=["use_external_asset", "use_source_asset"]) def single_auto_observe_asset_graph(request): @observable_source_asset(auto_observe_interval_minutes=30) - def asset1(): - ... + def asset1(): ... observable = create_external_asset_from_source_asset(asset1) if request.param else asset1 asset_graph = InternalAssetGraph.from_assets([observable]) @@ -105,8 +103,7 @@ def test_single_observable_source_asset_prior_recent_observe_requests( def test_reconcile(): @observable_source_asset(auto_observe_interval_minutes=30) - def asset1(): - ... + def asset1(): ... asset_graph = InternalAssetGraph.from_assets([asset1]) instance = DagsterInstance.ephemeral() diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py index 0b22f1561985..35088614fb46 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_check_decorator.py @@ -57,8 +57,7 @@ def _check(): def test_asset_check_with_prefix(): @asset(key_prefix="prefix") - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1) def my_check(): @@ -69,8 +68,7 @@ def my_check(): def test_asset_check_input_with_prefix(): @asset(key_prefix="prefix") - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1) def my_check(asset1): @@ -81,8 +79,7 @@ def my_check(asset1): def test_execute_asset_and_check(): @asset - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1, description="desc") def check1(context: AssetExecutionContext): @@ -160,8 +157,7 @@ def check1(): def test_execute_check_and_asset_in_separate_run(): @asset - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1, description="desc") def check1(context: AssetExecutionContext): @@ -195,8 +191,7 @@ def check1(context: AssetExecutionContext): def test_execute_check_and_unrelated_asset(): @asset - def asset2(): - ... + def asset2(): ... @asset_check(asset="asset1", description="desc") def check1(): @@ -251,16 +246,14 @@ def asset1_check(): def test_asset_check_separate_op_downstream_still_executes(): @asset - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1) def asset1_check(context: AssetExecutionContext): return AssetCheckResult(passed=False) @asset(deps=[asset1]) - def asset2(): - ... + def asset2(): ... result = execute_assets_and_checks(assets=[asset1, asset2], asset_checks=[asset1_check]) assert result.success @@ -278,16 +271,14 @@ def asset2(): def test_blocking_check_skip_downstream(): @asset - def asset1(): - ... + def asset1(): ... @asset_check(asset=asset1, blocking=True) def check1(context: AssetExecutionContext): return AssetCheckResult(passed=False) @asset(deps=[asset1]) - def asset2(): - ... + def asset2(): ... result = execute_assets_and_checks( assets=[asset1, asset2], asset_checks=[check1], raise_on_error=False @@ -319,8 +310,7 @@ def check1(context: AssetExecutionContext): return AssetCheckResult(passed=False) @asset(deps=[asset1]) - def asset2(): - ... + def asset2(): ... result = execute_assets_and_checks( assets=[asset1, asset2], asset_checks=[check1], raise_on_error=False @@ -384,8 +374,7 @@ def handle_output(self, context, obj): def test_definitions_conflicting_checks(): def make_check(): @asset_check(asset="asset1") - def check1(context: AssetExecutionContext): - ... + def check1(context: AssetExecutionContext): ... return check1 @@ -495,12 +484,10 @@ def my_check(context: AssetExecutionContext): def test_job_only_execute_checks_downstream_of_selected_assets(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset_check(asset=asset1) def check1(): @@ -530,8 +517,7 @@ def test_asset_not_provided(): with pytest.raises(Exception): @asset_check(description="desc") - def check1(): - ... + def check1(): ... def test_managed_input(): @@ -549,8 +535,7 @@ def load_input(self, context): assert context.asset_key == asset1.key return 4 - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... assert check1.name == "check1" assert check1.asset_key == asset1.key @@ -571,8 +556,7 @@ def test_multiple_managed_inputs(): ): @asset_check(asset="asset1", description="desc") - def check1(asset1, asset2): - ... + def check1(asset1, asset2): ... def test_managed_input_with_context(): diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_decorator_with_check_specs.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_decorator_with_check_specs.py index 51f9263c43c4..0e1f310529b6 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_decorator_with_check_specs.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_asset_decorator_with_check_specs.py @@ -219,8 +219,7 @@ def asset1(): yield AssetCheckResult(passed=False) @asset(deps=[asset1]) - def asset2(): - ... + def asset2(): ... result = materialize(assets=[asset1, asset2]) assert result.success @@ -278,8 +277,7 @@ def test_duplicate_checks_same_asset(): AssetCheckSpec("check1", asset="asset1", description="desc2"), ] ) - def asset1(): - ... + def asset1(): ... def test_check_wrong_asset(): @@ -293,8 +291,7 @@ def test_check_wrong_asset(): AssetCheckSpec("check1", asset="other_asset", description="desc1"), ] ) - def asset1(): - ... + def asset1(): ... def test_multi_asset_with_check(): diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_op.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_op.py index 928aa581297b..4c8c0dbdb0c5 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_op.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_op.py @@ -1628,8 +1628,7 @@ def test_none_annotated_input(): with pytest.raises(DagsterInvalidDefinitionError, match="is annotated with Nothing"): @op - def op1(input1: None): - ... + def op1(input1: None): ... def test_default_code_version(): diff --git a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_source_asset_decorator.py b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_source_asset_decorator.py index dfd18a21b203..6d3b6d73b605 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_source_asset_decorator.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/decorators_tests/test_source_asset_decorator.py @@ -54,29 +54,25 @@ def observable_asset_no_context(): def test_key_and_name_args(): @observable_source_asset(key=["apple", "banana"]) - def key_specified(): - ... + def key_specified(): ... assert key_specified.key == AssetKey(["apple", "banana"]) assert key_specified.op.name == "apple__banana" @observable_source_asset(key_prefix=["apple", "banana"]) - def key_prefix_specified(): - ... + def key_prefix_specified(): ... assert key_prefix_specified.key == AssetKey(["apple", "banana", "key_prefix_specified"]) assert key_prefix_specified.op.name == "apple__banana__key_prefix_specified" @observable_source_asset(name="peach") - def name_specified(): - ... + def name_specified(): ... assert name_specified.key == AssetKey(["peach"]) assert name_specified.op.name == "peach" @observable_source_asset(key_prefix=["apple", "banana"], name="peach") - def key_prefix_and_name_specified(): - ... + def key_prefix_and_name_specified(): ... assert key_prefix_and_name_specified.key == AssetKey(["apple", "banana", "peach"]) assert key_prefix_and_name_specified.op.name == "apple__banana__peach" @@ -87,8 +83,7 @@ def key_prefix_and_name_specified(): ): @observable_source_asset(key_prefix=["apple", "banana"], key=["peach", "nectarine"]) - def key_prefix_and_key_specified(): - ... + def key_prefix_and_key_specified(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -96,15 +91,13 @@ def key_prefix_and_key_specified(): ): @observable_source_asset(name=["peach"], key=["peach", "nectarine"]) - def name_and_key_specified(): - ... + def name_and_key_specified(): ... def test_op_tags(): tags = {"foo": "bar"} @observable_source_asset(op_tags=tags) - def op_tags_specified(): - ... + def op_tags_specified(): ... assert op_tags_specified.op.tags == tags diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_selection.py b/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_selection.py index af8e4cfd68b3..894437fd5c61 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_selection.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_selection.py @@ -13,13 +13,11 @@ @asset -def asset1(): - ... +def asset1(): ... @asset -def asset2(): - ... +def asset2(): ... @asset_check(asset=asset1) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_spec.py b/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_spec.py index 1eb9bb3f8631..8de459738388 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_spec.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_asset_check_spec.py @@ -10,8 +10,7 @@ def test_coerce_asset_key(): def test_asset_def(): @asset - def foo(): - ... + def foo(): ... assert AssetCheckSpec(asset=foo, name="check1").asset_key == AssetKey("foo") diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_definitions_class.py b/python_modules/dagster/dagster_tests/definitions_tests/test_definitions_class.py index 8d468068d2c8..dbceaa722305 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_definitions_class.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_definitions_class.py @@ -552,8 +552,7 @@ def test_unresolved_partitioned_asset_schedule(): partitions_def = DailyPartitionsDefinition(start_date="2020-01-01") @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... job1 = define_asset_job("job1") schedule1 = build_schedule_from_partitioned_job(job1) @@ -571,16 +570,13 @@ def asset1(): def test_bare_executor(): @asset - def an_asset(): - ... + def an_asset(): ... class DummyExecutor(Executor): - def execute(self, plan_context, execution_plan): - ... + def execute(self, plan_context, execution_plan): ... @property - def retries(self): - ... + def retries(self): ... executor_inst = DummyExecutor() diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_external_assets.py b/python_modules/dagster/dagster_tests/definitions_tests/test_external_assets.py index 37a9971f81ef..f917b5de4862 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_external_assets.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_external_assets.py @@ -88,8 +88,7 @@ def test_invalid_external_asset_creation() -> None: def test_normal_asset_materializeable() -> None: @asset - def an_asset() -> None: - ... + def an_asset() -> None: ... assert an_asset.is_executable diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_multi_partitions.py b/python_modules/dagster/dagster_tests/definitions_tests/test_multi_partitions.py index caff0acf5ca3..0f87f3e55662 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_multi_partitions.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_multi_partitions.py @@ -601,8 +601,7 @@ def a(a): second_partition_key = MultiPartitionKey({"time": "2020-01-02", "abc": "a"}) class MyIOManager(IOManager): - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... def load_input(self, context): assert context.asset_key.path[-1] == "a" diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_repository_definition.py b/python_modules/dagster/dagster_tests/definitions_tests/test_repository_definition.py index fead1f1ff997..62c5a787f2c7 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_repository_definition.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_repository_definition.py @@ -1277,8 +1277,7 @@ def test_scheduled_partitioned_asset_job(): partitions_def = DailyPartitionsDefinition(start_date="2022-06-06") @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -1392,16 +1391,13 @@ def the_repo(): def test_base_jobs(): @asset - def asset1(): - ... + def asset1(): ... @asset(partitions_def=StaticPartitionsDefinition(["a", "b", "c"])) - def asset2(): - ... + def asset2(): ... @asset(partitions_def=StaticPartitionsDefinition(["x", "y", "z"])) - def asset3(): - ... + def asset3(): ... @repository def repo(): @@ -1419,12 +1415,10 @@ def repo(): def test_auto_materialize_sensors_do_not_conflict(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @repository def repo(): @@ -1438,12 +1432,10 @@ def repo(): def test_auto_materialize_sensors_incomplete_cover(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @repository def repo(): @@ -1456,12 +1448,10 @@ def repo(): def test_auto_materialize_sensors_conflict(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -1483,12 +1473,10 @@ def test_invalid_asset_selection(): source_asset = SourceAsset("source_asset") @asset - def asset1(): - ... + def asset1(): ... @sensor(asset_selection=[source_asset, asset1]) - def sensor1(): - ... + def sensor1(): ... Definitions(assets=[source_asset, asset1], sensors=[sensor1]) diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_sensor.py b/python_modules/dagster/dagster_tests/definitions_tests/test_sensor.py index c46ce37a6d5e..f294c7b01907 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_sensor.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_sensor.py @@ -45,16 +45,13 @@ def test_direct_sensor_definition_instantiation(): def test_coerce_to_asset_selection(): @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(): - ... + def asset2(): ... @asset - def asset3(): - ... + def asset3(): ... assets = [asset1, asset2, asset3] diff --git a/python_modules/dagster/dagster_tests/definitions_tests/test_sensor_invocation.py b/python_modules/dagster/dagster_tests/definitions_tests/test_sensor_invocation.py index 46cfdc665e14..6f09bccd101e 100644 --- a/python_modules/dagster/dagster_tests/definitions_tests/test_sensor_invocation.py +++ b/python_modules/dagster/dagster_tests/definitions_tests/test_sensor_invocation.py @@ -371,8 +371,7 @@ def my_repo(): def test_multi_asset_sensor_with_source_assets() -> None: # upstream_asset1 exists in another repository @asset(partitions_def=DailyPartitionsDefinition(start_date="2023-03-01")) - def upstream_asset1(): - ... + def upstream_asset1(): ... upstream_asset1_source = SourceAsset( key=upstream_asset1.key, @@ -380,8 +379,7 @@ def upstream_asset1(): ) @asset() - def downstream_asset(upstream_asset1): - ... + def downstream_asset(upstream_asset1): ... @multi_asset_sensor( monitored_assets=[ diff --git a/python_modules/dagster/dagster_tests/execution_tests/pipes_tests/test_subprocess.py b/python_modules/dagster/dagster_tests/execution_tests/pipes_tests/test_subprocess.py index d5bd1c5974ef..b823eb75dd55 100644 --- a/python_modules/dagster/dagster_tests/execution_tests/pipes_tests/test_subprocess.py +++ b/python_modules/dagster/dagster_tests/execution_tests/pipes_tests/test_subprocess.py @@ -589,8 +589,7 @@ def sample_job(): def test_pipes_expected_materialization(): - def script_fn(): - ... + def script_fn(): ... @asset def missing_mat_result(context: OpExecutionContext, pipes_client: PipesSubprocessClient): @@ -662,8 +661,7 @@ def test_bad_user_message(): def script_fn(): from dagster_pipes import open_dagster_pipes - class Cursed: - ... + class Cursed: ... with open_dagster_pipes() as pipes: pipes.report_custom_message(Cursed()) diff --git a/python_modules/dagster/dagster_tests/execution_tests/test_data_versions.py b/python_modules/dagster/dagster_tests/execution_tests/test_data_versions.py index 1e3bc4a175d9..728a3dfc1095 100644 --- a/python_modules/dagster/dagster_tests/execution_tests/test_data_versions.py +++ b/python_modules/dagster/dagster_tests/execution_tests/test_data_versions.py @@ -67,8 +67,7 @@ def test_single_asset(): @asset - def asset1(): - ... + def asset1(): ... instance = DagsterInstance.ephemeral() mat1, mat2 = materialize_twice([asset1], asset1, instance) @@ -77,8 +76,7 @@ def asset1(): def test_single_versioned_asset(): @asset(code_version="abc") - def asset1(): - ... + def asset1(): ... instance = DagsterInstance.ephemeral() mat1, mat2 = materialize_twice([asset1], asset1, instance) @@ -89,8 +87,7 @@ def test_source_asset_non_versioned_asset(): source1 = SourceAsset("source1") @asset - def asset1(source1): - ... + def asset1(source1): ... instance = DagsterInstance.ephemeral() mat1, mat2 = materialize_twice([source1, asset1], asset1, instance) @@ -101,8 +98,7 @@ def test_source_asset_versioned_asset(): source1 = SourceAsset("source1") @asset(code_version="abc") - def asset1(source1): - ... + def asset1(source1): ... instance = DagsterInstance.ephemeral() @@ -114,8 +110,7 @@ def test_source_asset_non_versioned_asset_deps(): source1 = SourceAsset("source1") @asset(deps=[source1]) - def asset1(): - ... + def asset1(): ... instance = DagsterInstance.ephemeral() @@ -127,12 +122,10 @@ def test_versioned_after_unversioned(): source1 = SourceAsset("source1") @asset - def asset1(source1): - ... + def asset1(source1): ... @asset(code_version="abc") - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [source1, asset1, asset2] instance = DagsterInstance.ephemeral() @@ -151,12 +144,10 @@ def test_versioned_after_versioned(): source1 = SourceAsset("source1") @asset(code_version="abc") - def asset1(source1): - ... + def asset1(source1): ... @asset(code_version="xyz") - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [source1, asset1, asset2] instance = DagsterInstance.ephemeral() @@ -173,12 +164,10 @@ def test_unversioned_after_versioned(): source1 = SourceAsset("source1") @asset(code_version="abc") - def asset1(source1): - ... + def asset1(source1): ... @asset - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [source1, asset1, asset2] instance = DagsterInstance.ephemeral() @@ -268,12 +257,10 @@ def source1(_context): return DataVersion(str(x)) @asset(code_version="abc") - def asset1(source1): - ... + def asset1(source1): ... @asset(code_version="xyz") - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [source1, asset1, asset2] with instance_for_test() as instance: @@ -330,8 +317,7 @@ def asset2(asset1): # Simulate updating an asset with a new code version @asset(name="asset1", code_version="def") - def asset1_v2(source1): - ... + def asset1_v2(source1): ... all_assets_v2 = [source1, asset1_v2, asset2] @@ -354,12 +340,10 @@ def asset1_v2(source1): ] @asset - def asset3(): - ... + def asset3(): ... @asset(name="asset2", code_version="xyz") - def asset2_v2(asset3): - ... + def asset2_v2(asset3): ... all_assets_v3 = [source1, asset1_v2, asset2_v2, asset3] @@ -383,12 +367,10 @@ def asset2_v2(asset3): def test_stale_status_no_code_versions() -> None: @asset - def asset1(): - ... + def asset1(): ... @asset - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [asset1, asset2] with instance_for_test() as instance: @@ -425,12 +407,10 @@ def asset2(asset1): def test_stale_status_redundant_upstream_materialization() -> None: @asset(code_version="abc") - def asset1(): - ... + def asset1(): ... @asset - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [asset1, asset2] with instance_for_test() as instance: @@ -460,12 +440,10 @@ def asset1(context): return {key: randint(0, 100) for key in keys} @asset - def asset2(asset1): - ... + def asset2(asset1): ... @asset - def asset3(asset1): - ... + def asset3(asset1): ... all_assets = [asset1, asset2, asset3] with instance_for_test() as instance: @@ -505,12 +483,10 @@ def test_stale_status_partitions_disabled_code_versions() -> None: partitions_def = StaticPartitionsDefinition(["foo"]) @asset(code_version="1", partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @asset(code_version="1", partitions_def=partitions_def) - def asset2(asset1): - ... + def asset2(asset1): ... all_assets = [asset1, asset2] with instance_for_test() as instance: @@ -521,8 +497,7 @@ def asset2(asset1): assert status_resolver.get_status(asset2.key, "foo") == StaleStatus.FRESH @asset(code_version="2", partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... all_assets = [asset1, asset2] status_resolver = get_stale_status_resolver(instance, all_assets) @@ -541,12 +516,10 @@ def asset1(config: AssetConfig): return Output(config.value, data_version=DataVersion(str(config.value))) @asset(partitions_def=partitions_def) - def asset2(asset1): - ... + def asset2(asset1): ... @asset - def asset3(asset1): - ... + def asset3(asset1): ... all_assets = [asset1, asset2, asset3] with instance_for_test() as instance: @@ -603,8 +576,7 @@ def asset1(config: AssetConfig): return Output(1, data_version=DataVersion(str(config.value))) @asset(code_version="1") - def asset2(asset1): - ... + def asset2(asset1): ... @asset(partitions_def=partitions_def, code_version="1") def asset3(asset2): @@ -824,16 +796,13 @@ def source1(_context): return DataVersion(str(x)) @asset(code_version="1") - def asset1(source1): - ... + def asset1(source1): ... @asset(code_version="1") - def asset2(asset1): - ... + def asset2(asset1): ... @asset(code_version="1") - def asset3(asset2): - ... + def asset3(asset2): ... with instance_for_test() as instance: all_assets = [source1, asset1, asset2, asset3] @@ -845,8 +814,7 @@ def asset3(asset2): # Simulate updating an asset with a new code version @asset(name="asset1", code_version="2") - def asset1_v2(source1): - ... + def asset1_v2(source1): ... all_assets = [source1, asset1_v2, asset2, asset3] status_resolver = get_stale_status_resolver(instance, all_assets) @@ -883,8 +851,7 @@ def asset1_v2(source1): # Simulate updating an asset with a new code version @asset(name="asset3", code_version="2") - def asset3_v2(asset2): - ... + def asset3_v2(asset2): ... all_assets = [source1, asset1_v2, asset2, asset3_v2] status_resolver = get_stale_status_resolver(instance, all_assets) @@ -905,16 +872,13 @@ def asset1(): return Output(x, data_version=DataVersion(str(x))) @asset - def asset2(asset1): - ... + def asset2(asset1): ... @asset - def asset3(asset1): - ... + def asset3(asset1): ... @asset - def asset4(asset2, asset3): - ... + def asset4(asset2, asset3): ... with instance_for_test() as instance: all_assets = [asset1, asset2, asset3, asset4] @@ -929,8 +893,7 @@ def asset4(asset2, asset3): # Test dedup from updated code version @asset(name="asset1", code_version="2") - def asset1_v2(): - ... + def asset1_v2(): ... all_assets = [asset1_v2, asset2, asset3, asset4] status_resolver = get_stale_status_resolver(instance, all_assets) diff --git a/python_modules/dagster/dagster_tests/general_tests/test_repository.py b/python_modules/dagster/dagster_tests/general_tests/test_repository.py index 1d6c5a70c3e9..c1ed3a865742 100644 --- a/python_modules/dagster/dagster_tests/general_tests/test_repository.py +++ b/python_modules/dagster/dagster_tests/general_tests/test_repository.py @@ -1,5 +1,4 @@ -"""Repository of test jobs. -""" +"""Repository of test jobs.""" import pytest from dagster import ( @@ -127,8 +126,7 @@ def my_io_manager(): return MyIOManager() @asset - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -154,8 +152,7 @@ def my_io_manager(context): return MyIOManager(context.resource_config["key"]) @asset - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -183,8 +180,7 @@ def my_io_manager(): return MyIOManager() @asset - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -211,12 +207,10 @@ def my_io_manager(): return MyIOManager() @asset - def asset1(): - ... + def asset1(): ... @asset(metadata={"return": 20}) - def asset2(): - ... + def asset2(): ... @repository def repo(): diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_asset_events.py b/python_modules/dagster/dagster_tests/storage_tests/test_asset_events.py index 4efbc957b9c9..a5a7726eb80e 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_asset_events.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_asset_events.py @@ -45,8 +45,7 @@ def load_input(self, context): return 1 @asset - def before(): - ... + def before(): ... @asset def after(before): diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_asset_value_loader.py b/python_modules/dagster/dagster_tests/storage_tests/test_asset_value_loader.py index 607c78a9a66e..f73af474cb27 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_asset_value_loader.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_asset_value_loader.py @@ -22,8 +22,7 @@ def test_single_asset(): @asset(io_manager_key="my_io_manager", metadata={"a": "b"}) - def asset1(): - ... + def asset1(): ... class MyIOManager(IOManager): def handle_output(self, context, obj): @@ -124,8 +123,7 @@ def my_io_manager(): return MyIOManager() @asset(io_manager_key="my_io_manager") - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -168,12 +166,10 @@ def other_resource(): return "apple" @asset(io_manager_key="io_manager1") - def asset1(): - ... + def asset1(): ... @asset(io_manager_key="io_manager2") - def asset2(): - ... + def asset2(): ... @repository def repo(): @@ -229,8 +225,7 @@ def my_io_manager(): return MyIOManager() @asset(partitions_def=DailyPartitionsDefinition(start_date="2020-01-01")) - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -280,8 +275,7 @@ def my_io_manager(context): return MyIOManager(context.resource_config["key"]) @asset - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -310,8 +304,7 @@ def my_io_manager(): return MyIOManager() @asset - def asset1(): - ... + def asset1(): ... @repository def repo(): @@ -348,8 +341,7 @@ def first_order(context): return "bar" @asset(io_manager_key="the_io_manager") - def asset1(): - ... + def asset1(): ... @repository def repo(): diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_db_io_manager.py b/python_modules/dagster/dagster_tests/storage_tests/test_db_io_manager.py index cdde0038a1f4..68ad63a26883 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_db_io_manager.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_db_io_manager.py @@ -526,8 +526,7 @@ def test_default_load_type(): output_context = build_output_context(asset_key=asset_key, resource_config=resource_config) @asset - def asset1(): - ... + def asset1(): ... input_context = MagicMock( upstream_output=output_context, diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_fs_io_manager.py b/python_modules/dagster/dagster_tests/storage_tests/test_fs_io_manager.py index db2fd5eac586..0013de547a1e 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_fs_io_manager.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_fs_io_manager.py @@ -320,8 +320,7 @@ def description(self) -> str: partitions_def = DailyPartitionsDefinition(start_date="2020-02-01") @asset(partitions_def=partitions_def) - def asset1(): - ... + def asset1(): ... @asset( partitions_def=partitions_def, diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_input_manager.py b/python_modules/dagster/dagster_tests/storage_tests/test_input_manager.py index fde66c08ad79..bffabed3ca42 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_input_manager.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_input_manager.py @@ -356,8 +356,7 @@ def load_input(self, context): return 2 - def handle_output(self, context, obj): - ... + def handle_output(self, context, obj): ... materialize([upstream]) output = materialize( diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_io_manager.py b/python_modules/dagster/dagster_tests/storage_tests/test_io_manager.py index a838ef7f84bc..f8dc47fdd86d 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_io_manager.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_io_manager.py @@ -967,12 +967,10 @@ def handle_output(self, context, obj): my_io_manager = MyIOManager() @op(out=Out(Nothing)) - def op1(): - ... + def op1(): ... @op(ins={"input1": In(Nothing)}) - def op2(): - ... + def op2(): ... @job(resource_defs={"io_manager": IOManagerDefinition.hardcoded_io_manager(my_io_manager)}) def job1(): @@ -1005,12 +1003,10 @@ def handle_output(self, context, obj): my_io_manager = MyIOManager() @op(out=Out(Nothing)) - def op1(): - ... + def op1(): ... @op - def op2(_input1): - ... + def op2(_input1): ... @job(resource_defs={"io_manager": IOManagerDefinition.hardcoded_io_manager(my_io_manager)}) def job1(): diff --git a/python_modules/dagster/dagster_tests/storage_tests/test_io_manager_asset_metadata.py b/python_modules/dagster/dagster_tests/storage_tests/test_io_manager_asset_metadata.py index 823d650f9299..a58ac69af88a 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/test_io_manager_asset_metadata.py +++ b/python_modules/dagster/dagster_tests/storage_tests/test_io_manager_asset_metadata.py @@ -17,8 +17,7 @@ def materialize_expect_metadata(assets_def: AssetsDefinition): @asset(ins={key.path[-1]: AssetIn(key) for key in assets_def.keys}) - def downstream_asset(**kwargs): - ... + def downstream_asset(**kwargs): ... class MyIOManager(IOManager): def handle_output(self, context, obj): @@ -36,16 +35,14 @@ def load_input(self, context): def test_asset_with_metadata(): @asset(metadata={"fruit": "apple"}) - def basic_asset_with_metadata(): - ... + def basic_asset_with_metadata(): ... materialize_expect_metadata(basic_asset_with_metadata) def test_with_attributes_metadata(): @asset - def basic_asset_without_metadata(): - ... + def basic_asset_without_metadata(): ... materialize_expect_metadata( basic_asset_without_metadata.with_attributes( @@ -56,16 +53,14 @@ def basic_asset_without_metadata(): def test_multi_asset_with_metadata(): @multi_asset(outs={"asset1": AssetOut(metadata={"fruit": "apple"})}) - def multi_asset_with_metadata(): - ... + def multi_asset_with_metadata(): ... materialize_expect_metadata(multi_asset_with_metadata) def test_multi_asset_with_attributes_metadata(): @multi_asset(outs={"asset1": AssetOut()}) - def multi_asset_without_metadata(): - ... + def multi_asset_without_metadata(): ... materialize_expect_metadata( multi_asset_without_metadata.with_attributes( @@ -76,8 +71,7 @@ def multi_asset_without_metadata(): def test_graph_asset_outer_metadata(): @op - def op_without_output_metadata(): - ... + def op_without_output_metadata(): ... @graph_multi_asset(outs={"asset1": AssetOut(metadata={"fruit": "apple"})}) def graph_with_outer_metadata(): @@ -88,8 +82,7 @@ def graph_with_outer_metadata(): def test_graph_asset_op_metadata(): @op(out=Out(metadata={"fruit": "apple"})) - def op_without_output_metadata(): - ... + def op_without_output_metadata(): ... @graph_multi_asset(outs={"asset1": AssetOut()}) def graph_without_metadata(): @@ -104,8 +97,7 @@ def graph_without_metadata(): def test_assets_definition_from_graph_metadata(): @op - def op_without_output_metadata(): - ... + def op_without_output_metadata(): ... @graph(out={"asset1": GraphOut()}) def graph_without_metadata(): diff --git a/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py b/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py index 99b56852dfc2..379fad342363 100644 --- a/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py +++ b/python_modules/dagster/dagster_tests/storage_tests/utils/event_log_storage.py @@ -4920,8 +4920,7 @@ def test_transaction( ) assert len(storage.get_logs_for_run(test_run_id)) == 1 - class BlowUp(Exception): - ... + class BlowUp(Exception): ... # now try to delete with pytest.raises(BlowUp): diff --git a/python_modules/dagster/dagster_tests/test_annotations.py b/python_modules/dagster/dagster_tests/test_annotations.py index 57fa0261d117..d14a933a83b6 100644 --- a/python_modules/dagster/dagster_tests/test_annotations.py +++ b/python_modules/dagster/dagster_tests/test_annotations.py @@ -223,8 +223,7 @@ def bar(cls): def test_deprecated_abstractmethod(decorators): class Foo: @compose_decorators(*decorators) - def bar(self): - ... + def bar(self): ... assert is_deprecated(Foo.bar) # __dict__ access to get descriptor @@ -232,8 +231,7 @@ def bar(self): def test_deprecated_class(): @deprecated_bound class Foo: - def bar(self): - ... + def bar(self): ... assert is_deprecated(Foo) @@ -259,8 +257,7 @@ class Foo(NamedTuple("_", [("bar", str)])): def test_deprecated_resource(): @deprecated_bound @resource - def foo(): - ... + def foo(): ... assert is_deprecated(foo) @@ -370,8 +367,7 @@ def bar(cls, baz=None): def test_deprecated_param_abstractmethod(decorators): class Foo: @compose_decorators(*decorators) - def bar(self, baz=None): - ... + def bar(self, baz=None): ... assert is_deprecated_param(Foo.bar, "baz") @@ -379,8 +375,7 @@ def bar(self, baz=None): def test_deprecated_param_class(): @deprecated_param_bound class Foo: - def __init__(self, baz=None): - ... + def __init__(self, baz=None): ... assert is_deprecated_param(Foo, "baz") @@ -394,8 +389,7 @@ def __init__(self, baz=None): def test_deprecated_param_named_tuple_class(): @deprecated_param_bound class Foo(NamedTuple("_", [("baz", str)])): - def __new__(cls, baz=None): - ... + def __new__(cls, baz=None): ... assert is_deprecated_param(Foo, "baz") @@ -519,8 +513,7 @@ def bar(cls): def test_experimental_abstractmethod(decorators): class Foo: @compose_decorators(*decorators) - def bar(self): - ... + def bar(self): ... assert is_experimental(Foo.bar) @@ -528,8 +521,7 @@ def bar(self): def test_experimental_class(): @experimental class Foo: - def bar(self): - ... + def bar(self): ... assert is_experimental(Foo) @@ -605,8 +597,7 @@ class Foo(NamedTuple("_", [("bar", str)])): def test_experimental_resource(): @experimental @resource - def foo(): - ... + def foo(): ... assert is_experimental(foo) @@ -704,8 +695,7 @@ def bar(cls, baz=None): def test_experimental_param_abstractmethod(decorators): class Foo: @compose_decorators(*decorators) - def bar(self, baz=None): - ... + def bar(self, baz=None): ... assert is_experimental_param(Foo.bar, "baz") @@ -713,8 +703,7 @@ def bar(self, baz=None): def test_experimental_param_class(): @experimental_param(param="baz") class Foo: - def __init__(self, baz=None): - ... + def __init__(self, baz=None): ... assert is_experimental_param(Foo, "baz") @@ -728,8 +717,7 @@ def __init__(self, baz=None): def test_experimental_param_named_tuple_class(): @experimental_param(param="baz") class Foo(NamedTuple("_", [("baz", str)])): - def __new__(cls, baz=None): - ... + def __new__(cls, baz=None): ... assert is_experimental_param(Foo, "baz") diff --git a/python_modules/dagster/setup.py b/python_modules/dagster/setup.py index 2d15b5332b2b..00d00e0790fe 100644 --- a/python_modules/dagster/setup.py +++ b/python_modules/dagster/setup.py @@ -164,7 +164,7 @@ def get_version() -> str: "types-toml", # version will be resolved against toml ], "ruff": [ - "ruff==0.2.0", + "ruff==0.3.0", ], }, entry_points={ diff --git a/python_modules/libraries/dagster-airbyte/dagster_airbyte/asset_defs.py b/python_modules/libraries/dagster-airbyte/dagster_airbyte/asset_defs.py index 2ab737438004..893ffb10c9c4 100644 --- a/python_modules/libraries/dagster-airbyte/dagster_airbyte/asset_defs.py +++ b/python_modules/libraries/dagster-airbyte/dagster_airbyte/asset_defs.py @@ -644,9 +644,9 @@ def __init__( # display in the UI self._partially_initialized_airbyte_instance = airbyte_resource_def # The processed copy is used to query the Airbyte instance - self._airbyte_instance: ( - AirbyteResource - ) = self._partially_initialized_airbyte_instance.process_config_and_initialize() + self._airbyte_instance: AirbyteResource = ( + self._partially_initialized_airbyte_instance.process_config_and_initialize() + ) else: self._partially_initialized_airbyte_instance = airbyte_resource_def( build_init_resource_context() diff --git a/python_modules/libraries/dagster-airflow/dagster_airflow/dagster_asset_factory.py b/python_modules/libraries/dagster-airflow/dagster_airflow/dagster_asset_factory.py index b8aef71d6ce5..ddb1b76f5a57 100644 --- a/python_modules/libraries/dagster-airflow/dagster_airflow/dagster_asset_factory.py +++ b/python_modules/libraries/dagster-airflow/dagster_airflow/dagster_asset_factory.py @@ -83,9 +83,9 @@ def find_upstream_dependency(node_name: str) -> None: asset_upstream_deps ) else: - internal_asset_deps[ - f"result_{normalized_name(dag.dag_id, task_id)}" - ] = asset_upstream_deps + internal_asset_deps[f"result_{normalized_name(dag.dag_id, task_id)}"] = ( + asset_upstream_deps + ) # add new upstream asset dependencies to the internal deps for asset_key in upstream_dependencies_by_asset_key: diff --git a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/conftest.py b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/conftest.py index d551a3981de8..d719134c7dd3 100644 --- a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/conftest.py +++ b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/conftest.py @@ -3,6 +3,7 @@ These make very heavy use of fixture dependency and scope. If you're unfamiliar with pytest fixtures, read: https://docs.pytest.org/en/latest/fixture.html. """ + import os IS_BUILDKITE = os.getenv("BUILDKITE") is not None diff --git a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_persistent_db/conftest.py b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_persistent_db/conftest.py index 7682fe3442e0..222c070bb590 100644 --- a/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_persistent_db/conftest.py +++ b/python_modules/libraries/dagster-airflow/dagster_airflow_tests/test_persistent_db/conftest.py @@ -3,6 +3,7 @@ These make very heavy use of fixture dependency and scope. If you're unfamiliar with pytest fixtures, read: https://docs.pytest.org/en/latest/fixture.html. """ + import importlib import time from datetime import datetime diff --git a/python_modules/libraries/dagster-aws/dagster_aws/emr/configs_spark.py b/python_modules/libraries/dagster-aws/dagster_aws/emr/configs_spark.py index 8aadb22d6be2..15bb4431adab 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/emr/configs_spark.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/emr/configs_spark.py @@ -7,7 +7,6 @@ """ - from dagster import Bool, Field, Float, IntSource, Permissive, StringSource diff --git a/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/log4j.py b/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/log4j.py index 68ec8b055d10..4b9880186a13 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/log4j.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/log4j.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Parse the log4j syslog format used by Hadoop.""" + import re from collections import namedtuple diff --git a/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/retry.py b/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/retry.py index bffb97a2acd8..eb7a23bf8b85 100644 --- a/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/retry.py +++ b/python_modules/libraries/dagster-aws/dagster_aws/utils/mrjob/retry.py @@ -21,6 +21,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Wrappers for gracefully retrying on error.""" + import logging import time from functools import partial diff --git a/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/executor.py b/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/executor.py index b7375385f368..a292cae7c98a 100644 --- a/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/executor.py +++ b/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/executor.py @@ -367,9 +367,9 @@ def _execute_step_k8s_job( "dagster/run-id": execute_step_args.run_id, } if dagster_run.external_job_origin: - labels[ - "dagster/code-location" - ] = dagster_run.external_job_origin.external_repository_origin.code_location_origin.location_name + labels["dagster/code-location"] = ( + dagster_run.external_job_origin.external_repository_origin.code_location_origin.location_name + ) job = construct_dagster_k8s_job( job_config, args, diff --git a/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/launcher.py b/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/launcher.py index ab4029f14876..835730f59f9b 100644 --- a/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/launcher.py +++ b/python_modules/libraries/dagster-celery-k8s/dagster_celery_k8s/launcher.py @@ -212,9 +212,9 @@ def launch_run(self, context: LaunchRunContext) -> None: "dagster/run-id": run.run_id, } if run.external_job_origin: - labels[ - "dagster/code-location" - ] = run.external_job_origin.external_repository_origin.code_location_origin.location_name + labels["dagster/code-location"] = ( + run.external_job_origin.external_repository_origin.code_location_origin.location_name + ) job = construct_dagster_k8s_job( job_config, diff --git a/python_modules/libraries/dagster-databricks/dagster_databricks/configs.py b/python_modules/libraries/dagster-databricks/dagster_databricks/configs.py index 784fd5c28735..e74959f4f0bb 100644 --- a/python_modules/libraries/dagster-databricks/dagster_databricks/configs.py +++ b/python_modules/libraries/dagster-databricks/dagster_databricks/configs.py @@ -9,6 +9,7 @@ - https://docs.databricks.com/dev-tools/api/latest/clusters.html - https://docs.databricks.com/dev-tools/api/latest/libraries.html """ + from typing import Any, Dict from dagster import ( diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt/cloud/asset_defs.py b/python_modules/libraries/dagster-dbt/dagster_dbt/cloud/asset_defs.py index 55c05f0b2718..d64a63783fad 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt/cloud/asset_defs.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt/cloud/asset_defs.py @@ -529,9 +529,9 @@ def _assets(context: AssetExecutionContext): + split_materialization_command[idx + 2 :] ) - job_commands[ - job_materialization_command_step - ] = f"{materialization_command} {' '.join(dbt_options)}".strip() + job_commands[job_materialization_command_step] = ( + f"{materialization_command} {' '.join(dbt_options)}".strip() + ) # Run the dbt Cloud job to rematerialize the assets. dbt_cloud_output = dbt_cloud.run_job_and_poll( diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_check_selection.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_check_selection.py index e30db1146444..555f8f4cf9ad 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_check_selection.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_check_selection.py @@ -21,8 +21,7 @@ def my_dbt_assets_fixture(test_asset_checks_manifest: Dict[str, Any]) -> AssetsD manifest=test_asset_checks_manifest, dagster_dbt_translator=dagster_dbt_translator_with_checks, ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... return my_dbt_assets diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_checks.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_checks.py index b8cb8f94ec3d..876cb8a67df2 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_checks.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_checks.py @@ -229,8 +229,7 @@ def __init__(self, test_arg: str): super().__init__() - class CustomDagsterDbtTranslator(DagsterDbtTranslator): - ... + class CustomDagsterDbtTranslator(DagsterDbtTranslator): ... class CustomDagsterDbtTranslatorWithPassThrough(DagsterDbtTranslator): def __init__(self, test_arg: str, *args, **kwargs): diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_decorator.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_decorator.py index d926e41ce478..ff0e3a757748 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_decorator.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_decorator.py @@ -47,8 +47,7 @@ def test_manifest_argument( ]: @dbt_assets(manifest=manifest_param) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.keys == { AssetKey(key) @@ -204,8 +203,7 @@ def test_selections( select=select, exclude=exclude, ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... expected_asset_keys = {AssetKey(key) for key in expected_dbt_resource_names} @@ -217,8 +215,7 @@ def my_dbt_assets(): @pytest.mark.parametrize("name", [None, "custom"]) def test_with_custom_name(test_jaffle_shop_manifest: Dict[str, Any], name: Optional[str]) -> None: @dbt_assets(manifest=test_jaffle_shop_manifest, name=name) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... expected_name = name or "my_dbt_assets" @@ -232,8 +229,7 @@ def test_partitions_def( test_jaffle_shop_manifest: Dict[str, Any], partitions_def: Optional[PartitionsDefinition] ) -> None: @dbt_assets(manifest=test_jaffle_shop_manifest, partitions_def=partitions_def) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.partitions_def == partitions_def @@ -243,8 +239,7 @@ def test_io_manager_key( test_jaffle_shop_manifest: Dict[str, Any], io_manager_key: Optional[str] ) -> None: @dbt_assets(manifest=test_jaffle_shop_manifest, io_manager_key=io_manager_key) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... expected_io_manager_key = DEFAULT_IO_MANAGER_KEY if io_manager_key is None else io_manager_key @@ -301,8 +296,7 @@ def get_freshness_policy(cls, _: Mapping[str, Any]) -> Optional[FreshnessPolicy] backfill_policy=backfill_policy, dagster_dbt_translator=CustomDagsterDbtTranslator(), ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.backfill_policy == expected_backfill_policy @@ -311,8 +305,7 @@ def test_op_tags(test_jaffle_shop_manifest: Dict[str, Any]): op_tags = {"a": "b", "c": "d"} @dbt_assets(manifest=test_jaffle_shop_manifest, op_tags=op_tags) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.op.tags == { **op_tags, @@ -321,8 +314,7 @@ def my_dbt_assets(): } @dbt_assets(manifest=test_jaffle_shop_manifest, op_tags=op_tags, select="raw_customers+") - def my_dbt_assets_with_select(): - ... + def my_dbt_assets_with_select(): ... assert my_dbt_assets_with_select.op.tags == { **op_tags, @@ -331,8 +323,7 @@ def my_dbt_assets_with_select(): } @dbt_assets(manifest=test_jaffle_shop_manifest, op_tags=op_tags, exclude="raw_customers+") - def my_dbt_assets_with_exclude(): - ... + def my_dbt_assets_with_exclude(): ... assert my_dbt_assets_with_exclude.op.tags == { **op_tags, @@ -347,8 +338,7 @@ def my_dbt_assets_with_exclude(): select="raw_customers+", exclude="customers", ) - def my_dbt_assets_with_select_and_exclude(): - ... + def my_dbt_assets_with_select_and_exclude(): ... assert my_dbt_assets_with_select_and_exclude.op.tags == { **op_tags, @@ -369,8 +359,7 @@ def my_dbt_assets_with_select_and_exclude(): manifest=test_jaffle_shop_manifest, op_tags={"dagster-dbt/select": "raw_customers+"}, ) - def select_tag(): - ... + def select_tag(): ... with pytest.raises( DagsterInvalidDefinitionError, @@ -384,8 +373,7 @@ def select_tag(): manifest=test_jaffle_shop_manifest, op_tags={"dagster-dbt/exclude": "raw_customers+"}, ) - def exclude_tag(): - ... + def exclude_tag(): ... def test_with_asset_key_replacements(test_jaffle_shop_manifest: Dict[str, Any]) -> None: @@ -397,8 +385,7 @@ def get_asset_key(cls, dbt_resource_props: Mapping[str, Any]) -> AssetKey: @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.keys_by_input_name == { "__subset_input__model_jaffle_shop_stg_customers": AssetKey(["prefix", "stg_customers"]), @@ -453,8 +440,7 @@ def get_partition_mapping( dagster_dbt_translator=CustomizedDagsterDbtTranslator(), partitions_def=DailyPartitionsDefinition(start_date="2023-10-01"), ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... dependencies_with_self_dependencies = { # Self dependency enabled with `+meta.dagster.has_self_dependency` @@ -486,8 +472,7 @@ def get_description(cls, dbt_resource_props: Mapping[str, Any]) -> str: @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... for description in my_dbt_assets.descriptions_by_key.values(): assert description == expected_description @@ -504,8 +489,7 @@ def get_metadata(cls, dbt_resource_props: Mapping[str, Any]) -> Mapping[str, Any @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... for metadata in my_dbt_assets.metadata_by_key.values(): assert metadata["customized"] == "metadata" @@ -522,8 +506,7 @@ def get_group_name(cls, dbt_resource_props: Mapping[str, Any]) -> Optional[str]: @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... for group in my_dbt_assets.group_names_by_key.values(): assert group == expected_group @@ -542,8 +525,7 @@ def get_freshness_policy( @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... for freshness_policy in my_dbt_assets.freshness_policies_by_key.values(): assert freshness_policy == expected_freshness_policy @@ -564,8 +546,7 @@ def get_auto_materialize_policy( @dbt_assets( manifest=test_jaffle_shop_manifest, dagster_dbt_translator=CustomizedDagsterDbtTranslator() ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... for auto_materialize_policy in my_dbt_assets.auto_materialize_policies_by_key.values(): assert auto_materialize_policy == expected_auto_materialize_policy @@ -573,8 +554,7 @@ def my_dbt_assets(): def test_dbt_meta_auto_materialize_policy(test_meta_config_manifest: Dict[str, Any]) -> None: @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... auto_materialize_policies = my_dbt_assets.auto_materialize_policies_by_key.values() assert auto_materialize_policies @@ -585,8 +565,7 @@ def my_dbt_assets(): def test_dbt_meta_freshness_policy(test_meta_config_manifest: Dict[str, Any]) -> None: @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... freshness_policies = my_dbt_assets.freshness_policies_by_key.values() assert freshness_policies @@ -599,8 +578,7 @@ def my_dbt_assets(): def test_dbt_meta_asset_key(test_meta_config_manifest: Dict[str, Any]) -> None: @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... # Assert that source asset keys are set properly. assert AssetKey(["customized", "source", "jaffle_shop", "main", "raw_customers"]) in set( @@ -617,8 +595,7 @@ def my_dbt_assets(): def test_dbt_config_group(test_meta_config_manifest: Dict[str, Any]) -> None: @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert my_dbt_assets.group_names_by_key == { AssetKey(["customers"]): "default", @@ -637,8 +614,7 @@ def my_dbt_assets(): def test_dbt_with_downstream_asset_via_definition(test_meta_config_manifest: Dict[str, Any]): @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... @asset(deps=[my_dbt_assets]) def downstream_of_dbt(): @@ -653,8 +629,7 @@ def downstream_of_dbt(): def test_dbt_with_downstream_asset(test_meta_config_manifest: Dict[str, Any]): @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... @asset(deps=[AssetKey("orders"), AssetKey(["customized", "staging", "payments"])]) def downstream_of_dbt(): @@ -731,8 +706,7 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource): } @asset(key_prefix="dagster", deps=["raw_customers"]) - def python_augmented_customers(): - ... + def python_augmented_customers(): ... defs = Definitions( assets=[my_dbt_assets, python_augmented_customers], @@ -817,8 +791,7 @@ def test_dbt_with_invalid_self_dependencies( ) as exc_info: @dbt_assets(manifest=test_asset_key_exceptions_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert expected_error_message in str(exc_info.value) @@ -850,7 +823,6 @@ def get_asset_key(cls, dbt_resource_props: Mapping[str, Any]) -> AssetKey: manifest=test_meta_config_manifest, dagster_dbt_translator=CustomDagsterDbtTranslator(), ) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... assert expected_error_message in str(exc_info.value) diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_selection.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_selection.py index 7c6c988cc744..d13cf5dab84a 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_selection.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_asset_selection.py @@ -144,8 +144,7 @@ def test_dbt_asset_selection( expected_asset_keys = {AssetKey(key) for key in expected_dbt_resource_names} @dbt_assets(manifest=test_jaffle_shop_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... asset_graph = InternalAssetGraph.from_assets([my_dbt_assets]) asset_selection = build_dbt_asset_selection( @@ -182,8 +181,7 @@ def test_dbt_asset_selection_manifest_argument( ]: @dbt_assets(manifest=manifest_param) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... asset_graph = InternalAssetGraph.from_assets([my_dbt_assets]) asset_selection = build_dbt_asset_selection([my_dbt_assets], dbt_select="fqn:*") diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_dependencies.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_dependencies.py index 3cb2f3b7b56c..dafee71c19ad 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_dependencies.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_dependencies.py @@ -9,8 +9,7 @@ @pytest.fixture(name="my_dbt_assets", scope="module") def my_dbt_assets_fixture(test_meta_config_manifest: Dict[str, Any]) -> AssetsDefinition: @dbt_assets(manifest=test_meta_config_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... return my_dbt_assets @@ -19,8 +18,7 @@ def test_asset_downstream_of_dbt_asset(my_dbt_assets: AssetsDefinition) -> None: upstream_asset_key = AssetKey(["orders"]) @asset(deps=[get_asset_key_for_model([my_dbt_assets], "orders")]) - def downstream_python_asset(): - ... + def downstream_python_asset(): ... assert upstream_asset_key in my_dbt_assets.keys_by_output_name.values() assert set(downstream_python_asset.keys_by_input_name.values()) == {upstream_asset_key} diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_schedules.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_schedules.py index 93d64f2dc90a..eb7cd780cc97 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_schedules.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_schedules.py @@ -52,8 +52,7 @@ def test_dbt_build_schedule( default_status: DefaultScheduleStatus, ) -> None: @dbt_assets(manifest=test_jaffle_shop_manifest) - def my_dbt_assets(): - ... + def my_dbt_assets(): ... test_daily_schedule = build_schedule_from_dbt_selection( [my_dbt_assets], diff --git a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/sample_results.py b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/sample_results.py index 93a927146157..fcc810dda0a8 100644 --- a/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/sample_results.py +++ b/python_modules/libraries/dagster-dbt/dagster_dbt_tests/legacy/sample_results.py @@ -1,5 +1,4 @@ -"""This file contains sample run results responses from different DBT versions and invocations. -""" +"""This file contains sample run results responses from different DBT versions and invocations.""" DBT_18_RUN_RESULTS_SAMPLE = { "results": [ diff --git a/python_modules/libraries/dagster-deltalake/dagster_deltalake/io_manager.py b/python_modules/libraries/dagster-deltalake/dagster_deltalake/io_manager.py index 461cb49c9bb2..f9a332c84f4d 100644 --- a/python_modules/libraries/dagster-deltalake/dagster_deltalake/io_manager.py +++ b/python_modules/libraries/dagster-deltalake/dagster_deltalake/io_manager.py @@ -159,8 +159,7 @@ def my_table_a(my_table: pd.DataFrame): @staticmethod @abstractmethod - def type_handlers() -> Sequence[DbTypeHandler]: - ... + def type_handlers() -> Sequence[DbTypeHandler]: ... @staticmethod def default_load_type() -> Optional[Type]: diff --git a/python_modules/libraries/dagster-duckdb/dagster_duckdb/io_manager.py b/python_modules/libraries/dagster-duckdb/dagster_duckdb/io_manager.py index be0d59c1c79b..6ac3c6461b57 100644 --- a/python_modules/libraries/dagster-duckdb/dagster_duckdb/io_manager.py +++ b/python_modules/libraries/dagster-duckdb/dagster_duckdb/io_manager.py @@ -236,8 +236,7 @@ def my_table_a(my_table: pd.DataFrame): @staticmethod @abstractmethod - def type_handlers() -> Sequence[DbTypeHandler]: - ... + def type_handlers() -> Sequence[DbTypeHandler]: ... @staticmethod def default_load_type() -> Optional[Type]: diff --git a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/test_asset_decorator.py b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/test_asset_decorator.py index 3d10336638bb..97a42485cb07 100644 --- a/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/test_asset_decorator.py +++ b/python_modules/libraries/dagster-embedded-elt/dagster_embedded_elt_tests/test_asset_decorator.py @@ -19,8 +19,7 @@ ) def test_replication_param_defs(replication_params: SlingReplicationParam): @sling_assets(replication_config=replication_params) - def my_sling_assets(): - ... + def my_sling_assets(): ... assert my_sling_assets.keys == { AssetKey.from_user_string(key) @@ -40,8 +39,7 @@ def test_disabled_asset(): __file__, "replication_configs/base_config_disabled/replication.yaml" ) ) - def my_sling_assets(): - ... + def my_sling_assets(): ... assert my_sling_assets.keys == { AssetKey.from_user_string(key) diff --git a/python_modules/libraries/dagster-fivetran/dagster_fivetran/asset_defs.py b/python_modules/libraries/dagster-fivetran/dagster_fivetran/asset_defs.py index 5dc700e2ac24..8c2d5a7fa673 100644 --- a/python_modules/libraries/dagster-fivetran/dagster_fivetran/asset_defs.py +++ b/python_modules/libraries/dagster-fivetran/dagster_fivetran/asset_defs.py @@ -328,16 +328,16 @@ def __init__( # display in the UI self._partially_initialized_fivetran_instance = fivetran_resource_def # The processed copy is used to query the Fivetran instance - self._fivetran_instance: ( - FivetranResource - ) = self._partially_initialized_fivetran_instance.process_config_and_initialize() + self._fivetran_instance: FivetranResource = ( + self._partially_initialized_fivetran_instance.process_config_and_initialize() + ) else: self._partially_initialized_fivetran_instance = fivetran_resource_def( build_init_resource_context() ) - self._fivetran_instance: ( - FivetranResource - ) = self._partially_initialized_fivetran_instance + self._fivetran_instance: FivetranResource = ( + self._partially_initialized_fivetran_instance + ) self._key_prefix = key_prefix self._connector_to_group_fn = connector_to_group_fn diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/io_manager.py b/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/io_manager.py index 0e7304d15cde..273adf0f3453 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/io_manager.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/bigquery/io_manager.py @@ -306,8 +306,7 @@ def my_table_a(my_table: pd.DataFrame) -> pd.DataFrame: @staticmethod @abstractmethod - def type_handlers() -> Sequence[DbTypeHandler]: - ... + def type_handlers() -> Sequence[DbTypeHandler]: ... @staticmethod def default_load_type() -> Optional[Type]: diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_cluster.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_cluster.py index c612c5206cb8..96357b5e9d6d 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_cluster.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_cluster.py @@ -7,7 +7,6 @@ """ - from dagster import Bool, Field, Int, Permissive, Shape, String from .types_dataproc_cluster import Component diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_job.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_job.py index 11f7f1f7d80a..63863d14a6f8 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_job.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/configs_dataproc_job.py @@ -7,7 +7,6 @@ """ - from dagster import Bool, Field, Int, Permissive, Shape, String diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_cluster.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_cluster.py index 8cebcb504800..6289e7009c87 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_cluster.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_cluster.py @@ -7,7 +7,6 @@ """ - from dagster import Enum, EnumValue Component = Enum( diff --git a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_job.py b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_job.py index c114e82bdc88..259f530242c0 100644 --- a/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_job.py +++ b/python_modules/libraries/dagster-gcp/dagster_gcp/dataproc/types_dataproc_job.py @@ -7,7 +7,6 @@ """ - from dagster import Enum, EnumValue Substate = Enum( diff --git a/python_modules/libraries/dagster-k8s/dagster_k8s/executor.py b/python_modules/libraries/dagster-k8s/dagster_k8s/executor.py index 9f8bc972ed48..4892f4d0ab5b 100644 --- a/python_modules/libraries/dagster-k8s/dagster_k8s/executor.py +++ b/python_modules/libraries/dagster-k8s/dagster_k8s/executor.py @@ -274,9 +274,9 @@ def launch_step(self, step_handler_context: StepHandlerContext) -> Iterator[Dags "dagster/run-id": step_handler_context.execute_step_args.run_id, } if run.external_job_origin: - labels[ - "dagster/code-location" - ] = run.external_job_origin.external_repository_origin.code_location_origin.location_name + labels["dagster/code-location"] = ( + run.external_job_origin.external_repository_origin.code_location_origin.location_name + ) job = construct_dagster_k8s_job( job_config=job_config, args=args, diff --git a/python_modules/libraries/dagster-k8s/dagster_k8s/launcher.py b/python_modules/libraries/dagster-k8s/dagster_k8s/launcher.py index dad8270128bb..06656011ca29 100644 --- a/python_modules/libraries/dagster-k8s/dagster_k8s/launcher.py +++ b/python_modules/libraries/dagster-k8s/dagster_k8s/launcher.py @@ -239,9 +239,9 @@ def _launch_k8s_job_with_args( "dagster/run-id": run.run_id, } if run.external_job_origin: - labels[ - "dagster/code-location" - ] = run.external_job_origin.external_repository_origin.code_location_origin.location_name + labels["dagster/code-location"] = ( + run.external_job_origin.external_repository_origin.code_location_origin.location_name + ) job = construct_dagster_k8s_job( job_config=job_config, diff --git a/python_modules/libraries/dagster-k8s/dagster_k8s/ops/k8s_job_op.py b/python_modules/libraries/dagster-k8s/dagster_k8s/ops/k8s_job_op.py index 16daff58cf80..38f64e7a727b 100644 --- a/python_modules/libraries/dagster-k8s/dagster_k8s/ops/k8s_job_op.py +++ b/python_modules/libraries/dagster-k8s/dagster_k8s/ops/k8s_job_op.py @@ -314,9 +314,9 @@ def execute_k8s_job( "dagster/run-id": context.dagster_run.run_id, } if context.dagster_run.external_job_origin: - labels[ - "dagster/code-location" - ] = context.dagster_run.external_job_origin.external_repository_origin.code_location_origin.location_name + labels["dagster/code-location"] = ( + context.dagster_run.external_job_origin.external_repository_origin.code_location_origin.location_name + ) job = construct_dagster_k8s_job( job_config=k8s_job_config, diff --git a/python_modules/libraries/dagster-mlflow/dagster_mlflow/resources.py b/python_modules/libraries/dagster-mlflow/dagster_mlflow/resources.py index 7d549dc4b83d..7b157df50af8 100644 --- a/python_modules/libraries/dagster-mlflow/dagster_mlflow/resources.py +++ b/python_modules/libraries/dagster-mlflow/dagster_mlflow/resources.py @@ -2,6 +2,7 @@ class. This resource provides an easy way to configure mlflow for logging various things from dagster runs. """ + import atexit import sys from itertools import islice diff --git a/python_modules/libraries/dagster-mlflow/dagster_mlflow_tests/test_resources.py b/python_modules/libraries/dagster-mlflow/dagster_mlflow_tests/test_resources.py index 7a3db30ca430..414a92ebc328 100644 --- a/python_modules/libraries/dagster-mlflow/dagster_mlflow_tests/test_resources.py +++ b/python_modules/libraries/dagster-mlflow/dagster_mlflow_tests/test_resources.py @@ -1,5 +1,4 @@ -"""Unit testing the Mlflow class. -""" +"""Unit testing the Mlflow class.""" import logging import random diff --git a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/base.py b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/base.py index 7bfa38ae4209..f10f60754070 100644 --- a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/base.py +++ b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/base.py @@ -183,8 +183,7 @@ def write_df_to_path( df: pl.DataFrame, path: "UPath", metadata: Optional[StorageMetadata] = None, - ): - ... + ): ... @abstractmethod def sink_df_to_path( @@ -193,28 +192,24 @@ def sink_df_to_path( df: pl.LazyFrame, path: "UPath", metadata: Optional[StorageMetadata] = None, - ): - ... + ): ... @overload @abstractmethod def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[None, False] - ) -> pl.LazyFrame: - ... + ) -> pl.LazyFrame: ... @overload @abstractmethod def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[True] - ) -> LazyFrameWithMetadata: - ... + ) -> LazyFrameWithMetadata: ... @abstractmethod def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Optional[bool] = False - ) -> Union[pl.LazyFrame, LazyFrameWithMetadata]: - ... + ) -> Union[pl.LazyFrame, LazyFrameWithMetadata]: ... # tmp fix until https://github.com/dagster-io/dagster/pull/19294 is merged def load_input(self, context: InputContext) -> Union[Any, Dict[str, Any]]: diff --git a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/delta.py b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/delta.py index 86126719b2c2..1dd83f94dede 100644 --- a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/delta.py +++ b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/delta.py @@ -254,14 +254,12 @@ def write_df_to_path( @overload def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[None, False] - ) -> pl.LazyFrame: - ... + ) -> pl.LazyFrame: ... @overload def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[True] - ) -> LazyFrameWithMetadata: - ... + ) -> LazyFrameWithMetadata: ... def scan_df_from_path( self, diff --git a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/parquet.py b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/parquet.py index d4372dfe1186..ab94ead9d869 100644 --- a/python_modules/libraries/dagster-polars/dagster_polars/io_managers/parquet.py +++ b/python_modules/libraries/dagster-polars/dagster_polars/io_managers/parquet.py @@ -265,14 +265,12 @@ def write_df_to_path( @overload def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[None, False] - ) -> pl.LazyFrame: - ... + ) -> pl.LazyFrame: ... @overload def scan_df_from_path( self, path: "UPath", context: InputContext, with_metadata: Literal[True] - ) -> LazyFrameWithMetadata: - ... + ) -> LazyFrameWithMetadata: ... def scan_df_from_path( self, diff --git a/python_modules/libraries/dagster-spark/dagster_spark/configs.py b/python_modules/libraries/dagster-spark/dagster_spark/configs.py index 94ecbd415c77..06a6f4c2fb0b 100644 --- a/python_modules/libraries/dagster-spark/dagster_spark/configs.py +++ b/python_modules/libraries/dagster-spark/dagster_spark/configs.py @@ -5,6 +5,7 @@ https://spark.apache.org/docs/latest/submitting-applications.html for a more in-depth summary of Spark deployment contexts and configuration. """ + from dagster import Field, StringSource from .configs_spark import spark_config diff --git a/python_modules/libraries/dagster-spark/dagster_spark/configs_spark.py b/python_modules/libraries/dagster-spark/dagster_spark/configs_spark.py index 8aadb22d6be2..15bb4431adab 100644 --- a/python_modules/libraries/dagster-spark/dagster_spark/configs_spark.py +++ b/python_modules/libraries/dagster-spark/dagster_spark/configs_spark.py @@ -7,7 +7,6 @@ """ - from dagster import Bool, Field, Float, IntSource, Permissive, StringSource