gcpdiag.queries.monitoring

48def period_aligned_now(period_seconds: int) -> str:
49  """Return a MQL date string for the current timestamp aligned to the given period.
50
51  This will return "now - now%period" in a MQL-parseable date string and is useful
52  to get stable results. See also: (internal)
53  """
54
55  now = time.time()
56  now -= now % period_seconds
57  return time.strftime('%Y/%m/%d-%H:%M:%S+00:00', time.gmtime(now))

 60class TimeSeriesCollection(collections.abc.Mapping):
 61  """A mapping that stores Cloud Monitoring time series data.
 62
 63  Each time series is identified by a set of labels stored as
 64  frozenset where the elements are strings 'label:value'. E.g.:
 65
 66      frozenset({'resource.cluster_name:regional',
 67                 'resource.container_name:dnsmasq'})
 68
 69  The frozenset is used as key to store the time series data. The data
 70  is a dictionary with the following fields:
 71
 72  - 'start_time': timestamp string (ISO format) for the earliest point
 73  - 'end_time': timestamp string (ISO format) for the latest point
 74  - 'values': point values in bi-dimensional array-like structure:
 75    [[val1_t0, val2_t0], [val1_t1, val2_t1], ...]. The first dimension of
 76    the array is time, and the second is the value columns (usually there will
 77    be only one). The points are sorted chronologically (most recent point is
 78    the latest in the list).
 79  """
 80
 81  _data: dict
 82
 83  def __init__(self):
 84    # In order to ease the retrieval and matching, we store
 85    # label:value pairs as strings in a frozenset object.
 86    self._data = {}
 87
 88  def __str__(self):
 89    return str(self._data)
 90
 91  def __repr__(self):
 92    return repr(self._data)
 93
 94  def add_api_response(self, resource_data):
 95    """Add results to an existing TimeSeriesCollection object.
 96
 97    The monitoring API returns paginated results, so we need to be able to add
 98    results to an existing TimeSeriesCollection object.
 99    """
100
101    if 'timeSeriesData' not in resource_data:
102      return
103
104    for ts in resource_data['timeSeriesData']:
105      # No data?
106      if (
107        not ts['pointData']
108        or 'values' not in ts['pointData'][0]
109        or not ts['pointData'][0]['values']
110      ):
111        continue
112
113      # Use frozenset of label:value pairs as key to store the data
114      labels_dict = {}
115      if 'labelValues' in ts:
116        for i, value in enumerate(ts['labelValues']):
117          label_name = resource_data['timeSeriesDescriptor']['labelDescriptors'][i]['key']
118          if 'stringValue' in value:
119            labels_dict[label_name] = value['stringValue']
120        labels_frozenset = frozenset(f'{k}:{v}' for k, v in labels_dict.items())
121      else:
122        labels_frozenset = frozenset()
123
124      ts_point_data = ts['pointData']
125      self._data[labels_frozenset] = {
126        'labels': labels_dict,
127        'start_time': ts_point_data[-1]['timeInterval']['startTime'],
128        'end_time': ts_point_data[0]['timeInterval']['endTime'],
129        'values': [
130          _gcp_typed_values_to_python_list(ts_point_data[i]['values'])
131          for i in reversed(range(len(ts_point_data)))
132        ],
133      }
134
135  def __getitem__(self, labels):
136    """Returns the time series identified by labels (frozenset)."""
137    return self._data[labels]
138
139  def __iter__(self):
140    return iter(self._data)
141
142  def __len__(self):
143    return len(self._data)
144
145  def keys(self):
146    return self._data.keys()
147
148  def items(self):
149    return self._data.items()
150
151  def values(self):
152    return self._data.values()

155def query(project_id: str, query_str: str) -> TimeSeriesCollection:
156  """Do a monitoring query in the specified project.
157
158  Note that the project can be either the project where the monitored resources
159  are, or a workspace host project, in which case you will get results for all
160  associated monitored projects.
161  """
162
163  time_series = TimeSeriesCollection()
164
165  mon_api = apis.get_api('monitoring', 'v3', project_id)
166  try:
167    request = (
168      mon_api.projects()
169      .timeSeries()
170      .query(name='projects/' + project_id, body={'query': query_str})
171    )
172
173    logging.debug('executing monitoring query (project: %s)', project_id)
174    logging.debug('query: %s', query_str)
175    pages = 0
176    start_time = datetime.datetime.now()
177    while request:
178      pages += 1
179      response = request.execute(num_retries=config.API_RETRIES)
180      time_series.add_api_response(response)
181      request = (
182        mon_api.projects()
183        .timeSeries()
184        .query_next(previous_request=request, previous_response=response)
185      )
186      if request:
187        logging.debug('still executing monitoring query (project: %s)', project_id)
188    end_time = datetime.datetime.now()
189    logging.debug('query run time: %s, pages: %d', end_time - start_time, pages)
190  except googleapiclient.errors.HttpError as err:
191    gcp_err = utils.GcpApiError(err)
192    # Ignore 502 because we get that when the monitoring query times out.
193    if gcp_err.status in [502]:
194      logging.warning('error executing monitoring query: %s', str(gcp_err.message))
195    else:
196      raise utils.GcpApiError(err) from err
197  return time_series

200def queryrange(
201  project_id: str, query_str: str, start_time: datetime.datetime, end_time: datetime.datetime
202):
203  """
204  Do a monitoring query during specific timeframe in the specified project.
205
206  Note that the project can be either the project where the monitored resources
207  are, or a workspace host project, in which case you will get results for all
208  associated monitored projects.
209
210  """
211
212  mon_api = apis.get_api('monitoring', 'v1', project_id)
213
214  try:
215    step = '1m'
216    start_time_str = start_time.isoformat(timespec='seconds').replace('+00:00', 'Z')
217    end_time_str = end_time.isoformat(timespec='seconds').replace('+00:00', 'Z')
218    request = (
219      mon_api.projects()
220      .location()
221      .prometheus()
222      .api()
223      .v1()
224      .query_range(
225        name=f'projects/{project_id}',
226        location='global',
227        body={'query': query_str, 'start': start_time_str, 'end': end_time_str, 'step': step},
228      )
229    )
230    response = request.execute(num_retries=config.API_RETRIES)
231  except googleapiclient.errors.HttpError as err:
232    gcp_err = utils.GcpApiError(err)
233    # Ignore 502 because we get that when the monitoring query times out.
234    if gcp_err.status in [502]:
235      logging.warning('error executing monitoring query: %s', str(gcp_err.message))
236    else:
237      raise utils.GcpApiError(err) from err
238  return response