Dapr binding Azure Blob storage downloading large file (over 1gb) getting Connection aborted #760
Description
Please provide us with the following information:
This issue is a: (mark with an x)
- [x bug report -> please search issues before submitting
- documentation issue or request
- regression (a behavior that used to work and stopped in a new release)
Issue description
Having the dapr blobstorage binding to download the blob, smaller files is fine via the following config
{"TimeStamp":"2023-05-25T17:34:32.6571603+00:00","Log":"raise ConnectionError(err, request=request)"}
{"TimeStamp":"2023-05-25T17:34:32.6572473+00:00","Log":"('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"}
Steps to reproduce
- enable dapr
- setup input_storage with
componentType: bindings.azure.blobstorage - fetch the blob from the storage
download_url = "http://localhost:" + daprPort + "/v1.0/bindings/blobinput"
downloadcontents = (
'{ "operation": "get", "metadata": { "blobName": "' + blobname + '" } }'
)
- increase the dapr-http-max-request-size
--dapr-http-max-request-size 1596(this is for upload, though I am getting the blob from storage), not sure if this one is relevant
Expected behavior [What you expected to happen.]
I should see something like this a status 200 from blob download
"TimeStamp":"2023-05-25T16:25:51.0715986+00:00","Log":"Blob File created uploaded: https://ndpi.blob.core.windows.net/azure-webjobs-hosts/blobreceipts/ndpi-etl-input/Host.Functions.ndpi_input_poc/"0x8DB5D3CA9665635"/input/carbon (1).png"}
{"TimeStamp":"2023-05-25T16:25:51.0716222+00:00","Log":"Blob Name: carbon (1).png"}
{"TimeStamp":"2023-05-25T16:25:51.0868155+00:00","Log":"Directory '/blobs/dicom-2023-05-25-162551' created."}
{"TimeStamp":"2023-05-25T16:25:51.1859386+00:00","Log":"File List in /blobs/dicom-2023-05-25-162551 folder"}
{"TimeStamp":"2023-05-25T16:25:51.1941166+00:00","Log":"2023-05-25T16:25:51.194116616Z carbon-(1).png"}
{"TimeStamp":"2023-05-25T16:25:51.2081148+00:00","Log":"'/blobs/dcm-2023-05-25-162551' created."}
{"TimeStamp":"2023-05-25T16:25:51.2081917+00:00","Log":"File List in /blobs/dcm-2023-05-25-162551 folder"}
{"TimeStamp":"2023-05-25T16:25:51.2185379+00:00","Log":"- - [25/May/2023 16:25:51] "POST /queueinput HTTP/1.1" 200 -"}
Actual behavior [What actually happened.]
{"TimeStamp":"2023-05-25T17:34:26.0917946+00:00","Log":"Message Received: {"topic":"/subscriptions/83476aec-4579-44a0-94da-74d17d5f75b5/resourceGroups/pathology-poc/providers/Microsoft.Storage/storageAccounts/ndpi","subject":"/blobServices/default/containers/azure-webjobs-hosts/blobs/blobreceipts/ndpi-etl-input/Host.Functions.ndpi_input_poc/\\"0x8DB5D45F9F397DE\\"/input/SPECTRA-S1-1-1.ndpi","eventType":"Microsoft.Storage.BlobCreated","id":"03fc61c2-b01e-003a-6b2e-8fe29806da12","data":{"api":"PutBlob","clientRequestId":"d4d17918-a7bc-4b16-8524-fb0db90f42a0","requestId":"03fc61c2-b01e-003a-6b2e-8fe298000000","eTag":"0x8DB5D45FB25DD02","contentType":"application/octet-stream","contentLength":0,"blobType":"BlockBlob","url":"https://ndpi.blob.core.windows.net/azure-webjobs-hosts/blobreceipts/ndpi-etl-input/Host.Functions.ndpi_input_poc/\\"0x8DB5D45F9F397DE\\"/input/SPECTRA-S1-1-1.ndpi","sequencer":"0000000000000000000000000000D6FB0000000000fc1011","storageDiagnostics":{"batchId":"07820cf1-7006-000a-002e-8f5c57000000"}},"dataVersion":"","metadataVersion":"1","eventTime":"2023-05-25T17:32:15.0756326Z"}"}
{"TimeStamp":"2023-05-25T17:34:26.0918704+00:00","Log":"Blob File created uploaded: https://ndpi.blob.core.windows.net/azure-webjobs-hosts/blobreceipts/ndpi-etl-input/Host.Functions.ndpi_input_poc/"0x8DB5D45F9F397DE"/input/SPECTRA-S1-1-1.ndpi"}
{"TimeStamp":"2023-05-25T17:34:26.0918836+00:00","Log":"Blob Name: SPECTRA-S1-1-1.ndpi"}
{"TimeStamp":"2023-05-25T17:34:26.1119735+00:00","Log":"Directory '/blobs/dicom-2023-05-25-173426' created."}
{"TimeStamp":"2023-05-25T17:34:32.6566991+00:00","Log":"17:34:32,655] ERROR in app: Exception on /queueinput [POST]"}
{"TimeStamp":"2023-05-25T17:34:32.6567486+00:00","Log":"(most recent call last):"}
{"TimeStamp":"2023-05-25T17:34:32.6567557+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen"}
{"TimeStamp":"2023-05-25T17:34:32.6567609+00:00","Log":"response = self._make_request("}
{"TimeStamp":"2023-05-25T17:34:32.6567657+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request"}
{"TimeStamp":"2023-05-25T17:34:32.6567708+00:00","Log":"response = conn.getresponse()"}
{"TimeStamp":"2023-05-25T17:34:32.6567756+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 454, in getresponse"}
{"TimeStamp":"2023-05-25T17:34:32.6567798+00:00","Log":"httplib_response = super().getresponse()"}
{"TimeStamp":"2023-05-25T17:34:32.6567845+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 1377, in getresponse"}
{"TimeStamp":"2023-05-25T17:34:32.6567894+00:00","Log":"response.begin()"}
{"TimeStamp":"2023-05-25T17:34:32.6568069+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 320, in begin"}
{"TimeStamp":"2023-05-25T17:34:32.6568122+00:00","Log":"version, status, reason = self._read_status()"}
{"TimeStamp":"2023-05-25T17:34:32.6568165+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 289, in _read_status"}
{"TimeStamp":"2023-05-25T17:34:32.6568208+00:00","Log":"raise RemoteDisconnected("Remote end closed connection without""}
{"TimeStamp":"2023-05-25T17:34:32.656825+00:00","Log":"Remote end closed connection without response"}
{"TimeStamp":"2023-05-25T17:34:32.6568479+00:00","Log":""}
{"TimeStamp":"2023-05-25T17:34:32.6568528+00:00","Log":"handling of the above exception, another exception occurred:"}
{"TimeStamp":"2023-05-25T17:34:32.6568762+00:00","Log":""}
{"TimeStamp":"2023-05-25T17:34:32.6568809+00:00","Log":"(most recent call last):"}
{"TimeStamp":"2023-05-25T17:34:32.6568885+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 486, in send"}
{"TimeStamp":"2023-05-25T17:34:32.6568928+00:00","Log":"resp = conn.urlopen("}
{"TimeStamp":"2023-05-25T17:34:32.6569004+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 844, in urlopen"}
{"TimeStamp":"2023-05-25T17:34:32.656905+00:00","Log":"retries = retries.increment("}
{"TimeStamp":"2023-05-25T17:34:32.6569123+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/util/retry.py", line 470, in increment"}
{"TimeStamp":"2023-05-25T17:34:32.6569202+00:00","Log":"raise reraise(type(error), error, _stacktrace)"}
{"TimeStamp":"2023-05-25T17:34:32.6569249+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/util/util.py", line 38, in reraise"}
{"TimeStamp":"2023-05-25T17:34:32.6569324+00:00","Log":"raise value.with_traceback(tb)"}
{"TimeStamp":"2023-05-25T17:34:32.6569402+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 790, in urlopen"}
{"TimeStamp":"2023-05-25T17:34:32.6569452+00:00","Log":"response = self._make_request("}
{"TimeStamp":"2023-05-25T17:34:32.6569529+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connectionpool.py", line 536, in _make_request"}
{"TimeStamp":"2023-05-25T17:34:32.6569573+00:00","Log":"response = conn.getresponse()"}
{"TimeStamp":"2023-05-25T17:34:32.6569644+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/urllib3/connection.py", line 454, in getresponse"}
{"TimeStamp":"2023-05-25T17:34:32.6569688+00:00","Log":"httplib_response = super().getresponse()"}
{"TimeStamp":"2023-05-25T17:34:32.6569761+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 1377, in getresponse"}
{"TimeStamp":"2023-05-25T17:34:32.6569806+00:00","Log":"response.begin()"}
{"TimeStamp":"2023-05-25T17:34:32.6569847+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 320, in begin"}
{"TimeStamp":"2023-05-25T17:34:32.656992+00:00","Log":"version, status, reason = self._read_status()"}
{"TimeStamp":"2023-05-25T17:34:32.6569963+00:00","Log":"File "/usr/local/lib/python3.9/http/client.py", line 289, in _read_status"}
{"TimeStamp":"2023-05-25T17:34:32.6570002+00:00","Log":"raise RemoteDisconnected("Remote end closed connection without""}
{"TimeStamp":"2023-05-25T17:34:32.6570048+00:00","Log":"('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"}
{"TimeStamp":"2023-05-25T17:34:32.6570086+00:00","Log":""}
{"TimeStamp":"2023-05-25T17:34:32.6570163+00:00","Log":"handling of the above exception, another exception occurred:"}
{"TimeStamp":"2023-05-25T17:34:32.6570203+00:00","Log":""}
{"TimeStamp":"2023-05-25T17:34:32.6570276+00:00","Log":"(most recent call last):"}
{"TimeStamp":"2023-05-25T17:34:32.6570318+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 2190, in wsgi_app"}
{"TimeStamp":"2023-05-25T17:34:32.657039+00:00","Log":"response = self.full_dispatch_request()"}
{"TimeStamp":"2023-05-25T17:34:32.6570436+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1486, in full_dispatch_request"}
{"TimeStamp":"2023-05-25T17:34:32.6570509+00:00","Log":"rv = self.handle_user_exception(e)"}
{"TimeStamp":"2023-05-25T17:34:32.6570592+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1484, in full_dispatch_request"}
{"TimeStamp":"2023-05-25T17:34:32.6570636+00:00","Log":"rv = self.dispatch_request()"}
{"TimeStamp":"2023-05-25T17:34:32.657071+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/flask/app.py", line 1469, in dispatch_request"}
{"TimeStamp":"2023-05-25T17:34:32.6570755+00:00","Log":"return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)"}
{"TimeStamp":"2023-05-25T17:34:32.657083+00:00","Log":"File "/app/app.py", line 55, in incoming"}
{"TimeStamp":"2023-05-25T17:34:32.6570906+00:00","Log":"r = requests.post(download_url, downloadcontents)"}
{"TimeStamp":"2023-05-25T17:34:32.6570951+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 115, in post"}
{"TimeStamp":"2023-05-25T17:34:32.6571027+00:00","Log":"return request("post", url, data=data, json=json, **kwargs)"}
{"TimeStamp":"2023-05-25T17:34:32.6571103+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/api.py", line 59, in request"}
{"TimeStamp":"2023-05-25T17:34:32.657124+00:00","Log":"return session.request(method=method, url=url, **kwargs)"}
{"TimeStamp":"2023-05-25T17:34:32.657129+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 589, in request"}
{"TimeStamp":"2023-05-25T17:34:32.6571371+00:00","Log":"resp = self.send(prep, **send_kwargs)"}
{"TimeStamp":"2023-05-25T17:34:32.6571448+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/sessions.py", line 703, in send"}
{"TimeStamp":"2023-05-25T17:34:32.6571493+00:00","Log":"r = adapter.send(request, **kwargs)"}
{"TimeStamp":"2023-05-25T17:34:32.6571562+00:00","Log":"File "/usr/local/lib/python3.9/site-packages/requests/adapters.py", line 501, in send"}
{"TimeStamp":"2023-05-25T17:34:32.6571603+00:00","Log":"raise ConnectionError(err, request=request)"}
{"TimeStamp":"2023-05-25T17:34:32.6572473+00:00","Log":"('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"}
{"TimeStamp":"2023-05-25T17:34:32.6576654+00:00","Log":"- - [25/May/2023 17:34:32] "POST /queueinput HTTP/1.1" 500 -"}