Big Query RowIterator offset error #18601

285 3406
570 3406
855 3406
1140 3406
---------------------------------------------------------------------------
BadRequest                                Traceback (most recent call last)
Cell In[54], line 18
     14 rows = []
     15 while True:
     16     page = job.result(start_index=offset, max_results=page_size)
     17     total_rows = page.total_rows
---> 18     rows.extend(list(page))
     19     print(len(rows), total_rows)
     20     offset += page.num_results
     21     if page.num_results < page_size:

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\page_iterator.py:208, in Iterator._items_iter(self)
    206 def _items_iter(self):
    207     """Iterator for each item returned."""
--> 208     for page in self._page_iter(increment=False):
    209         for item in page:
    210             self.num_results += 1

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\page_iterator.py:250, in Iterator._page_iter(self, increment)
    248     self.num_results += page.num_items
    249 yield page
--> 250 page = self._next_page()

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\page_iterator.py:373, in HTTPIterator._next_page(self)
    366 """Get the next page in the iterator.
    367 
    368 Returns:
    369     Optional[Page]: The next page in the iterator or :data:`None` if
    370         there are no pages left.
    371 """
    372 if self._has_next_page():
--> 373     response = self._get_next_page_response()
    374     items = response.get(self._items_key, ())
    375     page = Page(self, items, self.item_to_value, raw_page=response)

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\cloud\bigquery\table.py:2171, in RowIterator._get_next_page_response(self)
   2168     if self.page_number and "startIndex" in params:
   2169         del params_copy["startIndex"]
-> 2171 return self.api_request(
   2172     method=self._HTTP_METHOD, path=self.path, query_params=params_copy
   2173 )

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\cloud\bigquery\client.py:917, in Client._call_api(self, retry, span_name, span_attributes, job_ref, headers, **kwargs)
    912     with create_span(
    913         name=span_name, attributes=span_attributes, client=self, job_ref=job_ref
    914     ):
    915         return call()
--> 917 return call()

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\retry\retry_unary.py:295, in Retry.__call__.<locals>.retry_wrapped_func(*args, **kwargs)
    291 target = functools.partial(func, *args, **kwargs)
    292 sleep_generator = exponential_sleep_generator(
    293     self._initial, self._maximum, multiplier=self._multiplier
    294 )
--> 295 return retry_target(
    296     target,
    297     self._predicate,
    298     sleep_generator,
    299     timeout=self._timeout,
    300     on_error=on_error,
    301 )

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\retry\retry_unary.py:157, in retry_target(target, predicate, sleep_generator, timeout, on_error, exception_factory, **kwargs)
    153 # pylint: disable=broad-except
    154 # This function explicitly must deal with broad exceptions.
    155 except Exception as exc:
    156     # defer to shared logic for handling errors
--> 157     next_sleep = _retry_error_helper(
    158         exc,
    159         deadline,
    160         sleep_iter,
    161         error_list,
    162         predicate,
    163         on_error,
    164         exception_factory,
    165         timeout,
    166     )
    167     # if exception not raised, sleep before next attempt
    168     time.sleep(next_sleep)

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\retry\retry_base.py:215, in _retry_error_helper(exc, deadline, sleep_iterator, error_list, predicate_fn, on_error_fn, exc_factory_fn, original_timeout)
    209 if not predicate_fn(exc):
    210     final_exc, source_exc = exc_factory_fn(
    211         error_list,
    212         RetryFailureReason.NON_RETRYABLE_ERROR,
    213         original_timeout,
    214     )
--> 215     raise final_exc from source_exc
    216 if on_error_fn is not None:
    217     on_error_fn(exc)

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\api_core\retry\retry_unary.py:148, in retry_target(target, predicate, sleep_generator, timeout, on_error, exception_factory, **kwargs)
    146 while True:
    147     try:
--> 148         result = target()
    149         if inspect.isawaitable(result):
    150             warnings.warn(_ASYNC_RETRY_WARNING)

File ~\AppData\Local\miniforge3\envs\test-big-query\Lib\site-packages\google\cloud\_http\__init__.py:509, in JSONConnection.api_request(self, method, path, query_params, data, content_type, headers, api_base_url, api_version, expect_json, _target_object, timeout, extra_api_info)
    497 response = self._make_request(
    498     method=method,
    499     url=url,
   (...)    505     extra_api_info=extra_api_info,
    506 )
    508 if not 200 <= response.status_code < 300:
--> 509     raise exceptions.from_http_response(response)
    511 if expect_json and response.content:
    512     return response.json()

BadRequest: 400 GET https://bigquery.googleapis.com/bigquery/v2/projects/gcp-wow-people-de-lab-dev/queries/d3576736-a9dc-4958-b31c-05abd63aff38?pageToken=BFNORWQYUEAQAAASA4EAAEEAQCAAKGQGBD7AUEE5AIQLBLQVJKTAECTXBISAUGLHMNYC253PO4WXAZLPOBWGKLLEMUWWYYLCFVSGK5QRYJQGXWCDAAAAAERJL42GEYJVGA4GGZLGGFRWIMJWG4YTEODBMQ2TCNJQG43WINDGHFQTSZJVGE3GGMZSGQNCIY3CMQ2DQYRQMMWTCMLEMQWTIODFHAWTSOLGMYWWKYRTGM4DMMJZG43DOZASIRQW433OGFQWEMJTGU2DMYJYHEYDIMBXME2WIMJUMI2TAOLCGBRDAMBTMY4TMYTBGY4WCN3DHA4TANTBGM2WGZTBMVQWKZLGG4YDIYZZGNRTENA2MUYWCYRRGM2TINTBHA4TANBQG5QTKZBRGRRDKMBZMIYGEMBQGNTDSNTCME3DSYJXMM4DSMBWMEZTKY3GMFSWCZLFMY3TANDDHEZWGMRUENTDEZDEGE3TQZJNGYZTOYJNGRRWMNBNMIYDKMRNMMZTGZDEGJTDMMJZGY2A%3D%3D%3D%3D&maxResults=19&fields=jobReference%2CtotalRows%2CpageToken%2Crows&location=US&startIndex=1140&formatOptions.useInt64Timestamp=True&prettyPrint=false: When using a page token, you cannot specify an arbitrary startIndex.