gcpdiag.queries.composer

Queries related to Composer.
class Environment(gcpdiag.models.Resource):
 29class Environment(models.Resource):
 30  """Represents Composer environment"""
 31
 32  _resource_data: dict
 33
 34  def __init__(self, project_id: str, resource_data: dict):
 35    super().__init__(project_id)
 36    self._resource_data = resource_data
 37    self.region, self.name = self.parse_full_path()
 38    self.version_pattern = re.compile(r'composer-(.*)-airflow-(.*)')
 39
 40  @property
 41  def webserver_cpu(self) -> Optional[float]:
 42    return get_path(
 43      self._resource_data,
 44      ('config', 'workloadsConfig', 'webServer', 'cpu'),
 45      default=None,
 46    )
 47
 48  @property
 49  def webserver_memory_gb(self) -> Optional[float]:
 50    return get_path(
 51      self._resource_data,
 52      ('config', 'workloadsConfig', 'webServer', 'memoryGb'),
 53      default=None,
 54    )
 55
 56  @property
 57  def worker_cpu(self) -> float:
 58    return get_path(
 59      self._resource_data,
 60      ('config', 'workloadsConfig', 'worker', 'cpu'),
 61      default=None,
 62    )
 63
 64  @property
 65  def worker_memory_gb(self) -> float:
 66    return get_path(
 67      self._resource_data,
 68      ('config', 'workloadsConfig', 'worker', 'memoryGb'),
 69      default=None,
 70    )
 71
 72  @property
 73  def worker_max_count(self) -> int:
 74    return get_path(
 75      self._resource_data,
 76      ('config', 'workloadsConfig', 'worker', 'maxCount'),
 77      default=None,
 78    )
 79
 80  @property
 81  def worker_concurrency(self) -> float:
 82    def default_value():
 83      airflow_version = self.airflow_version
 84
 85      if version.parse(airflow_version) < version.parse('2.3.3'):
 86        return 12 * self.worker_cpu
 87      else:
 88        return min(32, 12 * self.worker_cpu, 8 * self.worker_memory_gb)
 89
 90    return float(self.airflow_config_overrides.get('celery-worker_concurrency', default_value()))
 91
 92  @property
 93  def parallelism(self) -> float:
 94    return float(self.airflow_config_overrides.get('core-parallelism', 'inf'))
 95
 96  @property
 97  def composer_version(self) -> str:
 98    v = self.version_pattern.search(self.image_version)
 99    assert v is not None
100    return v.group(1)
101
102  @property
103  def airflow_version(self) -> str:
104    v = self.version_pattern.search(self.image_version)
105    assert v is not None
106    return v.group(2)
107
108  @property
109  def is_composer2(self) -> bool:
110    return self.composer_version.startswith('2')
111
112  @property
113  def full_path(self) -> str:
114    return self._resource_data['name']
115
116  @property
117  def state(self) -> str:
118    return self._resource_data['state']
119
120  @property
121  def image_version(self) -> str:
122    return self._resource_data['config']['softwareConfig']['imageVersion']
123
124  @property
125  def short_path(self) -> str:
126    return f'{self.project_id}/{self.region}/{self.name}'
127
128  @property
129  def airflow_config_overrides(self) -> dict:
130    return self._resource_data['config']['softwareConfig'].get('airflowConfigOverrides', {})
131
132  @property
133  def service_account(self) -> str:
134    sa = self._resource_data['config']['nodeConfig'].get('serviceAccount')
135    if sa is None:
136      # serviceAccount is marked as optional in REST API docs
137      # using a default GCE SA as a fallback
138      project_nr = crm.get_project(self.project_id).number
139      sa = f'{project_nr}-compute@developer.gserviceaccount.com'
140    return sa
141
142  def parse_full_path(self) -> Tuple[str, str]:
143    match = re.match(r'projects/[^/]*/locations/([^/]*)/environments/([^/]*)', self.full_path)
144    if not match:
145      raise RuntimeError(f"Can't parse full_path {self.full_path}")
146    return match.group(1), match.group(2)
147
148  def __str__(self) -> str:
149    return self.short_path
150
151  def is_private_ip(self) -> bool:
152    return self._resource_data['config']['privateEnvironmentConfig'].get(
153      'enablePrivateEnvironment', False
154    )
155
156  @property
157  def gke_cluster(self) -> str:
158    return self._resource_data['config']['gkeCluster']
159
160  @property
161  def num_schedulers(self) -> int:
162    return get_path(
163      self._resource_data,
164      ('config', 'workloadsConfig', 'scheduler', 'count'),
165      default=1,
166    )
167
168  @property
169  def scheduler_cpu(self) -> float:
170    return get_path(
171      self._resource_data,
172      ('config', 'workloadsConfig', 'scheduler', 'cpu'),
173      default=None,
174    )
175
176  @property
177  def scheduler_memory_gb(self) -> float:
178    return get_path(
179      self._resource_data,
180      ('config', 'workloadsConfig', 'scheduler', 'memoryGb'),
181      default=None,
182    )

Represents Composer environment

Environment(project_id: str, resource_data: dict)
34  def __init__(self, project_id: str, resource_data: dict):
35    super().__init__(project_id)
36    self._resource_data = resource_data
37    self.region, self.name = self.parse_full_path()
38    self.version_pattern = re.compile(r'composer-(.*)-airflow-(.*)')
version_pattern
webserver_cpu: Optional[float]
40  @property
41  def webserver_cpu(self) -> Optional[float]:
42    return get_path(
43      self._resource_data,
44      ('config', 'workloadsConfig', 'webServer', 'cpu'),
45      default=None,
46    )
webserver_memory_gb: Optional[float]
48  @property
49  def webserver_memory_gb(self) -> Optional[float]:
50    return get_path(
51      self._resource_data,
52      ('config', 'workloadsConfig', 'webServer', 'memoryGb'),
53      default=None,
54    )
worker_cpu: float
56  @property
57  def worker_cpu(self) -> float:
58    return get_path(
59      self._resource_data,
60      ('config', 'workloadsConfig', 'worker', 'cpu'),
61      default=None,
62    )
worker_memory_gb: float
64  @property
65  def worker_memory_gb(self) -> float:
66    return get_path(
67      self._resource_data,
68      ('config', 'workloadsConfig', 'worker', 'memoryGb'),
69      default=None,
70    )
worker_max_count: int
72  @property
73  def worker_max_count(self) -> int:
74    return get_path(
75      self._resource_data,
76      ('config', 'workloadsConfig', 'worker', 'maxCount'),
77      default=None,
78    )
worker_concurrency: float
80  @property
81  def worker_concurrency(self) -> float:
82    def default_value():
83      airflow_version = self.airflow_version
84
85      if version.parse(airflow_version) < version.parse('2.3.3'):
86        return 12 * self.worker_cpu
87      else:
88        return min(32, 12 * self.worker_cpu, 8 * self.worker_memory_gb)
89
90    return float(self.airflow_config_overrides.get('celery-worker_concurrency', default_value()))
parallelism: float
92  @property
93  def parallelism(self) -> float:
94    return float(self.airflow_config_overrides.get('core-parallelism', 'inf'))
composer_version: str
 96  @property
 97  def composer_version(self) -> str:
 98    v = self.version_pattern.search(self.image_version)
 99    assert v is not None
100    return v.group(1)
airflow_version: str
102  @property
103  def airflow_version(self) -> str:
104    v = self.version_pattern.search(self.image_version)
105    assert v is not None
106    return v.group(2)
is_composer2: bool
108  @property
109  def is_composer2(self) -> bool:
110    return self.composer_version.startswith('2')
full_path: str
112  @property
113  def full_path(self) -> str:
114    return self._resource_data['name']

Returns the full path of this resource.

Example: 'projects/gcpdiag-gke-1-9b90/zones/europe-west4-a/clusters/gke1'

state: str
116  @property
117  def state(self) -> str:
118    return self._resource_data['state']
image_version: str
120  @property
121  def image_version(self) -> str:
122    return self._resource_data['config']['softwareConfig']['imageVersion']
short_path: str
124  @property
125  def short_path(self) -> str:
126    return f'{self.project_id}/{self.region}/{self.name}'

Returns the short name for this resource.

Note that it isn't clear from this name what kind of resource it is.

Example: 'gke1'

airflow_config_overrides: dict
128  @property
129  def airflow_config_overrides(self) -> dict:
130    return self._resource_data['config']['softwareConfig'].get('airflowConfigOverrides', {})
service_account: str
132  @property
133  def service_account(self) -> str:
134    sa = self._resource_data['config']['nodeConfig'].get('serviceAccount')
135    if sa is None:
136      # serviceAccount is marked as optional in REST API docs
137      # using a default GCE SA as a fallback
138      project_nr = crm.get_project(self.project_id).number
139      sa = f'{project_nr}-compute@developer.gserviceaccount.com'
140    return sa
def parse_full_path(self) -> Tuple[str, str]:
142  def parse_full_path(self) -> Tuple[str, str]:
143    match = re.match(r'projects/[^/]*/locations/([^/]*)/environments/([^/]*)', self.full_path)
144    if not match:
145      raise RuntimeError(f"Can't parse full_path {self.full_path}")
146    return match.group(1), match.group(2)
def is_private_ip(self) -> bool:
151  def is_private_ip(self) -> bool:
152    return self._resource_data['config']['privateEnvironmentConfig'].get(
153      'enablePrivateEnvironment', False
154    )
gke_cluster: str
156  @property
157  def gke_cluster(self) -> str:
158    return self._resource_data['config']['gkeCluster']
num_schedulers: int
160  @property
161  def num_schedulers(self) -> int:
162    return get_path(
163      self._resource_data,
164      ('config', 'workloadsConfig', 'scheduler', 'count'),
165      default=1,
166    )
scheduler_cpu: float
168  @property
169  def scheduler_cpu(self) -> float:
170    return get_path(
171      self._resource_data,
172      ('config', 'workloadsConfig', 'scheduler', 'cpu'),
173      default=None,
174    )
scheduler_memory_gb: float
176  @property
177  def scheduler_memory_gb(self) -> float:
178    return get_path(
179      self._resource_data,
180      ('config', 'workloadsConfig', 'scheduler', 'memoryGb'),
181      default=None,
182    )
COMPOSER_REGIONS = ['asia-northeast2', 'us-central1', 'northamerica-northeast1', 'us-west3', 'southamerica-east1', 'us-east1', 'asia-northeast1', 'europe-west1', 'europe-west2', 'asia-northeast3', 'us-west4', 'asia-east2', 'europe-central2', 'europe-west6', 'us-west2', 'australia-southeast1', 'europe-west3', 'asia-south1', 'us-west1', 'us-east4', 'asia-southeast1']
@caching.cached_api_call
def get_environments( context: gcpdiag.models.Context) -> Iterable[Environment]:
231@caching.cached_api_call
232def get_environments(context: models.Context) -> Iterable[Environment]:
233  environments: List[Environment] = []
234  if not apis.is_enabled(context.project_id, 'composer'):
235    return environments
236  api = apis.get_api('composer', 'v1', context.project_id)
237
238  for env in _query_regions_envs(COMPOSER_REGIONS, api, context.project_id, context):
239    # projects/{projectId}/locations/{locationId}/environments/{environmentId}.
240    result = re.match(r'projects/[^/]+/locations/([^/]+)/environments/([^/]+)', env['name'])
241    if not result:
242      logging.error('invalid composer name: %s', env['name'])
243      continue
244    location = result.group(1)
245    labels = env.get('labels', {})
246    name = result.group(2)
247    if not context.match_project_resource(location=location, labels=labels, resource=name):
248      continue
249
250    environments.append(Environment(context.project_id, env))
251  return environments