gcpdiag.queries.composer
Queries related to Composer.
class
Environment(gcpdiag.models.Resource):
29class Environment(models.Resource): 30 """Represents Composer environment""" 31 32 _resource_data: dict 33 34 def __init__(self, project_id: str, resource_data: dict): 35 super().__init__(project_id) 36 self._resource_data = resource_data 37 self.region, self.name = self.parse_full_path() 38 self.version_pattern = re.compile(r'composer-(.*)-airflow-(.*)') 39 40 @property 41 def webserver_cpu(self) -> Optional[float]: 42 return get_path( 43 self._resource_data, 44 ('config', 'workloadsConfig', 'webServer', 'cpu'), 45 default=None, 46 ) 47 48 @property 49 def webserver_memory_gb(self) -> Optional[float]: 50 return get_path( 51 self._resource_data, 52 ('config', 'workloadsConfig', 'webServer', 'memoryGb'), 53 default=None, 54 ) 55 56 @property 57 def worker_cpu(self) -> float: 58 return get_path( 59 self._resource_data, 60 ('config', 'workloadsConfig', 'worker', 'cpu'), 61 default=None, 62 ) 63 64 @property 65 def worker_memory_gb(self) -> float: 66 return get_path( 67 self._resource_data, 68 ('config', 'workloadsConfig', 'worker', 'memoryGb'), 69 default=None, 70 ) 71 72 @property 73 def worker_max_count(self) -> int: 74 return get_path( 75 self._resource_data, 76 ('config', 'workloadsConfig', 'worker', 'maxCount'), 77 default=None, 78 ) 79 80 @property 81 def worker_concurrency(self) -> float: 82 def default_value(): 83 airflow_version = self.airflow_version 84 85 if version.parse(airflow_version) < version.parse('2.3.3'): 86 return 12 * self.worker_cpu 87 else: 88 return min(32, 12 * self.worker_cpu, 8 * self.worker_memory_gb) 89 90 return float(self.airflow_config_overrides.get('celery-worker_concurrency', default_value())) 91 92 @property 93 def parallelism(self) -> float: 94 return float(self.airflow_config_overrides.get('core-parallelism', 'inf')) 95 96 @property 97 def composer_version(self) -> str: 98 v = self.version_pattern.search(self.image_version) 99 assert v is not None 100 return v.group(1) 101 102 @property 103 def airflow_version(self) -> str: 104 v = self.version_pattern.search(self.image_version) 105 assert v is not None 106 return v.group(2) 107 108 @property 109 def is_composer2(self) -> bool: 110 return self.composer_version.startswith('2') 111 112 @property 113 def full_path(self) -> str: 114 return self._resource_data['name'] 115 116 @property 117 def state(self) -> str: 118 return self._resource_data['state'] 119 120 @property 121 def image_version(self) -> str: 122 return self._resource_data['config']['softwareConfig']['imageVersion'] 123 124 @property 125 def short_path(self) -> str: 126 return f'{self.project_id}/{self.region}/{self.name}' 127 128 @property 129 def airflow_config_overrides(self) -> dict: 130 return self._resource_data['config']['softwareConfig'].get('airflowConfigOverrides', {}) 131 132 @property 133 def service_account(self) -> str: 134 sa = self._resource_data['config']['nodeConfig'].get('serviceAccount') 135 if sa is None: 136 # serviceAccount is marked as optional in REST API docs 137 # using a default GCE SA as a fallback 138 project_nr = crm.get_project(self.project_id).number 139 sa = f'{project_nr}-compute@developer.gserviceaccount.com' 140 return sa 141 142 def parse_full_path(self) -> Tuple[str, str]: 143 match = re.match(r'projects/[^/]*/locations/([^/]*)/environments/([^/]*)', self.full_path) 144 if not match: 145 raise RuntimeError(f"Can't parse full_path {self.full_path}") 146 return match.group(1), match.group(2) 147 148 def __str__(self) -> str: 149 return self.short_path 150 151 def is_private_ip(self) -> bool: 152 return self._resource_data['config']['privateEnvironmentConfig'].get( 153 'enablePrivateEnvironment', False 154 ) 155 156 @property 157 def gke_cluster(self) -> str: 158 return self._resource_data['config']['gkeCluster'] 159 160 @property 161 def num_schedulers(self) -> int: 162 return get_path( 163 self._resource_data, 164 ('config', 'workloadsConfig', 'scheduler', 'count'), 165 default=1, 166 ) 167 168 @property 169 def scheduler_cpu(self) -> float: 170 return get_path( 171 self._resource_data, 172 ('config', 'workloadsConfig', 'scheduler', 'cpu'), 173 default=None, 174 ) 175 176 @property 177 def scheduler_memory_gb(self) -> float: 178 return get_path( 179 self._resource_data, 180 ('config', 'workloadsConfig', 'scheduler', 'memoryGb'), 181 default=None, 182 )
Represents Composer environment
worker_concurrency: float
80 @property 81 def worker_concurrency(self) -> float: 82 def default_value(): 83 airflow_version = self.airflow_version 84 85 if version.parse(airflow_version) < version.parse('2.3.3'): 86 return 12 * self.worker_cpu 87 else: 88 return min(32, 12 * self.worker_cpu, 8 * self.worker_memory_gb) 89 90 return float(self.airflow_config_overrides.get('celery-worker_concurrency', default_value()))
full_path: str
Returns the full path of this resource.
Example: 'projects/gcpdiag-gke-1-9b90/zones/europe-west4-a/clusters/gke1'
short_path: str
124 @property 125 def short_path(self) -> str: 126 return f'{self.project_id}/{self.region}/{self.name}'
Returns the short name for this resource.
Note that it isn't clear from this name what kind of resource it is.
Example: 'gke1'
service_account: str
132 @property 133 def service_account(self) -> str: 134 sa = self._resource_data['config']['nodeConfig'].get('serviceAccount') 135 if sa is None: 136 # serviceAccount is marked as optional in REST API docs 137 # using a default GCE SA as a fallback 138 project_nr = crm.get_project(self.project_id).number 139 sa = f'{project_nr}-compute@developer.gserviceaccount.com' 140 return sa
COMPOSER_REGIONS =
['asia-northeast2', 'us-central1', 'northamerica-northeast1', 'us-west3', 'southamerica-east1', 'us-east1', 'asia-northeast1', 'europe-west1', 'europe-west2', 'asia-northeast3', 'us-west4', 'asia-east2', 'europe-central2', 'europe-west6', 'us-west2', 'australia-southeast1', 'europe-west3', 'asia-south1', 'us-west1', 'us-east4', 'asia-southeast1']
@caching.cached_api_call
def
get_environments( context: gcpdiag.models.Context) -> Iterable[Environment]:
231@caching.cached_api_call 232def get_environments(context: models.Context) -> Iterable[Environment]: 233 environments: List[Environment] = [] 234 if not apis.is_enabled(context.project_id, 'composer'): 235 return environments 236 api = apis.get_api('composer', 'v1', context.project_id) 237 238 for env in _query_regions_envs(COMPOSER_REGIONS, api, context.project_id, context): 239 # projects/{projectId}/locations/{locationId}/environments/{environmentId}. 240 result = re.match(r'projects/[^/]+/locations/([^/]+)/environments/([^/]+)', env['name']) 241 if not result: 242 logging.error('invalid composer name: %s', env['name']) 243 continue 244 location = result.group(1) 245 labels = env.get('labels', {}) 246 name = result.group(2) 247 if not context.match_project_resource(location=location, labels=labels, resource=name): 248 continue 249 250 environments.append(Environment(context.project_id, env)) 251 return environments