![]() When the DAG structure is similar from one run to the next, it clarifies the unit of work and continuity. Project FocusĪirflow works best with workflows that are mostly static and slowly changing. The rich user interface makes it easy to visualize pipelines running in production, monitor progress, and troubleshoot issues when needed. Rich command line utilities make performing complex surgeries on DAGs a snap. The Airflow scheduler executes your tasks on an array of workers while following the specified dependencies. Use Airflow to author workflows as directed acyclic graphs (DAGs) of tasks. When workflows are defined as code, they become more maintainable, versionable, testable, and collaborative. Apache Airflow (or simply Airflow) is a platform to programmatically author, schedule, and monitor workflows. Nested (TaskSchema )) 97 total_entries = fields. params 79 return 80 81 82 def _get_is_mapped (obj ): 83 return isinstance (obj, MappedOperator ) 84 85 86 class TaskCollection (NamedTuple ): 87 """List of Tasks with metadata.""" 88 89 tasks : list 90 total_entries : int 91 92 93 class TaskCollectionSchema (Schema ): 94 """Schema for TaskCollection.""" 95 96 tasks = fields. operator_name 74 75 76 def _get_params (obj ): 77 """Get the Params defined in a Task.""" 78 params = obj. data if hasattr (result, "data" ) else result 70 71 72 def _get_operator_name (obj ): 73 return obj. Method ( "_get_is_mapped", dump_only = True ) 65 66 67 def _get_class_reference (obj ): 68 result = ClassReferenceSchema (). Method ( "_get_params", dump_only = True ) 64 is_mapped = fields. String (), dump_only = True ) 63 params = fields. Nested (DAGSchema, dump_only = True ) 62 downstream_task_ids = fields. String (), dump_only = True ) 61 sub_dag = fields. ![]() Number (dump_only = True ) 57 weight_rule = WeightRuleField (dump_only = True ) 58 ui_color = ColorField (dump_only = True ) 59 ui_fgcolor = ColorField (dump_only = True ) 60 template_fields = fields. Boolean (dump_only = True ) 56 priority_weight = fields. Nested (TimeDeltaSchema, dump_only = True ) 55 retry_exponential_backoff = fields. Nested (TimeDeltaSchema, dump_only = True ) 54 retry_delay = fields. Number (dump_only = True ) 53 execution_timeout = fields. String (dump_only = True ) 52 pool_slots = fields. String (dump_only = True ) 51 pool = fields. Number (dump_only = True ) 50 queue = fields. Boolean (dump_only = True ) 49 retries = fields. Boolean (dump_only = True ) 48 wait_for_downstream = fields. Nested (ClassReferenceSchema ), dump_only = True, attribute = "operator_extra_links" 46 ) 47 depends_on_past = fields. String (dump_only = True ) 44 extra_links = fields. DateTime (dump_only = True ) 43 trigger_rule = fields. DateTime (dump_only = True ) 42 end_date = fields. String (dump_only = True ) 41 start_date = fields. String (dump_only = True ) 40 owner = fields. Method ( "_get_operator_name", dump_only = True ) 39 task_id = fields. ![]() Method ( "_get_class_reference", dump_only = True ) 38 operator_name = fields. operator import Operator 32 33 34 class TaskSchema (Schema ): 35 """Task schema.""" 36 37 class_ref = fields. mappedoperator import MappedOperator 31 from airflow. dag_schema import DAGSchema 30 from airflow. common_schema import ( 24 ClassReferenceSchema, 25 ColorField, 26 TimeDeltaSchema, 27 WeightRuleField, 28 ) 29 from airflow. 17 from _future_ import annotations 18 19 from typing import NamedTuple 20 21 from marshmallow import Schema, fields 22 23 from airflow. See the License for the 15 # specific language governing permissions and limitations 16 # under the License. ![]() You may obtain a copy of the License at 8 # 9 # 10 # 11 # Unless required by applicable law or agreed to in writing, 12 # software distributed under the License is distributed on an 13 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 # KIND, either express or implied. The ASF licenses this file 5 # to you under the Apache License, Version 2.0 (the 6 # "License") you may not use this file except in compliance 7 # with the License. See the NOTICE file 3 # distributed with this work for additional information 4 # regarding copyright ownership. As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Python source code syntax highlighting (style: standard) with prefixed line numbers.Īlternatively you can here view or download the uninterpreted source code file.įor more information about "task_schema.py" see the Fossies "Dox" file reference documentation.ġ # Licensed to the Apache Software Foundation (ASF) under one 2 # or more contributor license agreements. ![]()
0 Comments
Leave a Reply. |
AuthorWrite something about yourself. No need to be fancy, just an overview. ArchivesCategories |