gcpdiag.queries.apis_utils

28def list_all(request,
29             next_function: Callable,
30             response_keyword='items') -> Iterator[Any]:
31  """Execute GCP API `request` and subsequently call `next_function` until
32  there are no more results. Assumes that it is a list method and that
33  the results are under a `items` key."""
34
35  while True:
36    try:
37      response = request.execute(num_retries=config.API_RETRIES)
38    except googleapiclient.errors.HttpError as err:
39      raise utils.GcpApiError(err) from err
40
41    # Empty lists are omitted in GCP API responses
42    if response_keyword in response:
43      yield from response[response_keyword]
44
45    request = next_function(previous_request=request,
46                            previous_response=response)
47    if request is None:
48      break

51def batch_list_all(api,
52                   requests: list,
53                   next_function: Callable,
54                   log_text: str,
55                   response_keyword='items'):
56  """Similar to list_all but using batch API except in TPC environment."""
57
58  if 'googleapis.com' not in requests[0].uri:
59    #  the api client library does not handle batch api calls for TPC yet, so
60    #  the batch is processed and collected one at a time in that case
61    for req in requests:
62      yield from list_all(req, next_function)
63  else:
64    yield from _original_batch(api, requests, next_function, log_text,
65                               response_keyword)

103def should_retry(resp_status):
104  if resp_status >= 500:
105    return True
106  if resp_status == 429:  # too many requests
107    return True
108  return False

111def get_nth_exponential_random_retry(n, random_pct, mutiplier, random_fn=None):
112  random_fn = random_fn or random.random
113  return (1 - random_fn() * random_pct) * mutiplier**n

116def batch_execute_all(api, requests: list):
117  """Execute all `requests` using the batch API and yield (request,response,exception)
118  tuples."""
119  # results: (request, result, exception) tuples
120  results: List[Tuple[Any, Optional[Any], Optional[Exception]]] = []
121  requests_todo = requests
122  requests_in_flight: List = []
123  retry_count = 0
124
125  def fetch_all_cb(request_id, response, exception):
126    try:
127      request = requests_in_flight[int(request_id)]
128    except (IndexError, ValueError, TypeError):
129      logging.debug(
130          'BUG: Cannot find request %r in list of pending requests, dropping request.',
131          request_id)
132      return
133
134    if exception:
135      if isinstance(exception, googleapiclient.errors.HttpError) and \
136        should_retry(exception.status_code) and \
137        retry_count < config.API_RETRIES:
138        logging.debug('received HTTP error status code %d from API, retrying',
139                      exception.status_code)
140        requests_todo.append(request)
141      else:
142        results.append((request, None, utils.GcpApiError(exception)))
143      return
144
145    if not response:
146      return
147
148    results.append((request, response, None))
149
150  while True:
151    requests_in_flight = requests_todo
152    requests_todo = []
153    results = []
154
155    # Do the batch API request
156    try:
157      batch = api.new_batch_http_request()
158      for i, req in enumerate(requests_in_flight):
159        batch.add(req, callback=fetch_all_cb, request_id=str(i))
160      batch.execute()
161    except (googleapiclient.errors.HttpError, httplib2.HttpLib2Error) as err:
162      if isinstance(err, googleapiclient.errors.HttpError):
163        error_msg = f'received HTTP error status code {err.status_code} from Batch API, retrying'
164      else:
165        error_msg = f'received exception from Batch API: {err}, retrying'
166      if (not isinstance(err, googleapiclient.errors.HttpError) or \
167          should_retry(err.status_code)) \
168          and retry_count < config.API_RETRIES:
169        logging.debug(error_msg)
170        requests_todo = requests_in_flight
171        results = []
172      else:
173        raise utils.GcpApiError(err) from err
174
175    # Yield results
176    yield from results
177
178    # If no requests_todo, means we are done.
179    if not requests_todo:
180      break
181
182    # for example: retry delay: 20% is random, progression: 1, 1.4, 2.0, 2.7, ... 28.9 (10 retries)
183    sleep_time = get_nth_exponential_random_retry(
184        n=retry_count,
185        random_pct=config.API_RETRY_SLEEP_RANDOMNESS_PCT,
186        mutiplier=config.API_RETRY_SLEEP_MULTIPLIER)
187    logging.debug('sleeping %.2f seconds before retry #%d', sleep_time,
188                  retry_count + 1)
189    time.sleep(sleep_time)
190    retry_count += 1