diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8bfc92e..8db1fcd 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -13,6 +13,8 @@ jobs: steps: - uses: actions/checkout@v6 + with: + lfs: true - name: Set up Python uses: actions/setup-python@v6 diff --git a/examples/api2-devdock-transloadit-assembly-upload/main.py b/examples/api2-devdock-transloadit-assembly-upload/main.py new file mode 100644 index 0000000..6f94c75 --- /dev/null +++ b/examples/api2-devdock-transloadit-assembly-upload/main.py @@ -0,0 +1,61 @@ +"""Upload to a Transloadit devdock Assembly using tus-py-client. + +This example is intentionally checked into the SDK repository. API2 owns the +scenario JSON and prepares the live Transloadit Assembly; this file only shows +ordinary tus-py-client usage against the injected TUS endpoint. +""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + scenario_bytes, + tus_url, + upload_metadata, + write_result, +) + + +def upload_with_tus(scenario, create_response): + upload_config = scenario["upload"] + endpoint_url = tus_url(upload_config, scenario, create_response) + content = scenario_bytes(upload_config) + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + uploader = tus.TusClient(endpoint_url).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail("TUS upload offset {}, expected {}".format(uploader.offset, len(content))) + + return uploader.url + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + upload_url = upload_with_tus(scenario, create_response) + write_result({"uploadUrl": upload_url}) + print( + "Python TUS SDK devdock scenario {} uploaded to {}".format( + scenario["scenarioId"], upload_url + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-abort-upload/main.py b/examples/api2-devdock-tus-abort-upload/main.py new file mode 100644 index 0000000..ea40d24 --- /dev/null +++ b/examples/api2-devdock-tus-abort-upload/main.py @@ -0,0 +1,155 @@ +"""Abort a TUS upload against the API2 devdock conformance server.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.exceptions import TusUploadAborted +from tusclient.fingerprint.interface import Fingerprint +from tusclient.storage.interface import Storage + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + bool_value, + conformance_input_options, + conformance_input_source_bytes, + fail, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +class MemoryStorage(Storage): + def __init__(self): + self.values = {} + + def get_item(self, key): + return self.values.get(key) + + def set_item(self, key, value): + self.values[key] = value + + def remove_item(self, key): + self.values.pop(key, None) + + +class FixedFingerprint(Fingerprint): + def __init__(self, fingerprint): + self.fingerprint = fingerprint + + def get_fingerprint(self, fs): + return self.fingerprint + + +def upload_and_abort(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + headers = object_value(input_options.get("headers", {}), "headers") + metadata = object_value(input_options["metadata"], "metadata") + override_patch_method = bool_value( + input_options.get("overridePatchMethod", False), + "overridePatchMethod", + ) + runtime_setup = object_value( + conformance_scenario["runtimeSetup"], + "conformanceScenario.runtimeSetup", + ) + abort_setup = object_value( + runtime_setup["abort"], + "conformanceScenario.runtimeSetup.abort", + ) + terminate_upload_on_abort = bool_value( + abort_setup["terminateUpload"], + "conformanceScenario.runtimeSetup.abort.terminateUpload", + ) + + client_ref = {} + uploader_ref = {} + + def on_abort_request(event): + active_client = client_ref.get("client") + active_uploader = uploader_ref.get("uploader") + if active_client is None: + fail("abort request observed before client was initialized") + active_client.abort_upload(active_uploader, False) + + with TusConformancePlanServer( + conformance_scenario, + endpoint_url, + on_abort_request=on_abort_request, + ) as conformance_server: + client = tus.TusClient(conformance_server.endpoint_url(), headers=headers) + client_ref["client"] = client + + storage = None + fingerprint = input_options.get("fingerprint") + uploader_options = {} + if fingerprint is not None: + storage = MemoryStorage() + uploader_options.update( + { + "fingerprinter": FixedFingerprint( + string_value(fingerprint, "fingerprint"), + ), + "store_url": True, + "url_storage": storage, + } + ) + + uploader = client.uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=metadata, + override_patch_method=override_patch_method, + **uploader_options, + ) + uploader_ref["uploader"] = uploader + + try: + uploader.upload() + except TusUploadAborted: + pass + else: + fail("abort scenario completed without TusUploadAborted") + + if terminate_upload_on_abort: + if not uploader.url: + fail("abort scenario requested termination before upload URL was known") + client.abort_upload(uploader, True) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["completionKind"] = "aborted" + result["errorCalled"] = False + result["successCalled"] = False + result["uploadUrl"] = ( + conformance_server.canonical_url(uploader.url) if uploader.url else None + ) + if storage is not None and fingerprint is not None: + result["storedUrlAfterAbort"] = storage.get_item(fingerprint) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_and_abort(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} aborted the upload".format( + scenario_id(scenario), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-creation-with-upload/main.py b/examples/api2-devdock-tus-creation-with-upload/main.py new file mode 100644 index 0000000..6828381 --- /dev/null +++ b/examples/api2-devdock-tus-creation-with-upload/main.py @@ -0,0 +1,71 @@ +"""Create a Transloadit devdock TUS upload with bytes in the creation request.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + scenario_bytes, + scenario_id, + tus_url, + upload_metadata, + write_result, +) + + +def upload_with_creation_body(scenario, create_response): + upload_config = scenario["upload"] + content = scenario_bytes(upload_config) + + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + if not upload_config["uploadDataDuringCreation"]: + fail("scenario does not enable uploadDataDuringCreation") + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response) + ).create_upload_with_data( + len(content), + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("creation-with-upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "creation-with-upload accepted {} bytes, expected {}".format( + uploader.offset, + len(content), + ) + ) + + return { + "acceptedBytes": uploader.offset, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_creation_body(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} uploaded during creation to {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-custom-request-headers/main.py b/examples/api2-devdock-tus-custom-request-headers/main.py new file mode 100644 index 0000000..f09362c --- /dev/null +++ b/examples/api2-devdock-tus-custom-request-headers/main.py @@ -0,0 +1,99 @@ +"""Send Transloadit devdock TUS custom request headers.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + scenario_bytes, + scenario_id, + tus_url, + upload_headers, + upload_metadata, + write_result, +) + + +def record_custom_request_headers(context, expected_headers): + observed = {} + for header_name, expected_value in expected_headers.items(): + actual_value = context.headers.get(header_name) + if actual_value != expected_value: + fail( + "custom request header {} expected {!r}, got {!r}".format( + header_name, + expected_value, + actual_value, + ) + ) + observed[header_name] = actual_value + return observed + + +def upload_with_custom_request_headers(scenario, create_response): + upload_config = scenario["upload"] + expected_headers = upload_headers(scenario) + content = scenario_bytes(upload_config) + headers_by_method = {} + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + def before_request(context): + if context.method in ("POST", "PATCH"): + headers_by_method[context.method] = record_custom_request_headers( + context, + expected_headers, + ) + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response), + headers=expected_headers, + request_hooks=RequestLifecycleHooks(before_request=before_request), + ).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("custom request headers upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "custom request headers upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + for method in ("POST", "PATCH"): + if method not in headers_by_method: + fail("custom request headers did not observe {} request".format(method)) + + return { + "headersByMethod": headers_by_method, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_custom_request_headers(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} sent custom request headers for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-deferred-length-upload/main.py b/examples/api2-devdock-tus-deferred-length-upload/main.py new file mode 100644 index 0000000..68e87a5 --- /dev/null +++ b/examples/api2-devdock-tus-deferred-length-upload/main.py @@ -0,0 +1,83 @@ +"""Create a Transloadit devdock TUS upload with deferred upload length.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + fixed_chunk_size_bytes, + load_scenario, + scenario_bytes, + scenario_id, + tus_url, + upload_metadata, + write_result, +) + + +def upload_with_deferred_length(scenario, create_response): + upload_config = scenario["upload"] + content = scenario_bytes(upload_config) + chunk_size = fixed_chunk_size_bytes(scenario) + + if not upload_config["uploadLengthDeferred"]: + fail("scenario does not enable uploadLengthDeferred") + if chunk_size <= len(content): + fail( + "deferred-length scenario chunk size {} must exceed payload size {}".format( + chunk_size, + len(content), + ) + ) + + uploader = tus.TusClient(tus_url(upload_config, scenario, create_response)).uploader( + file_stream=BytesIO(content), + chunk_size=chunk_size, + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + upload_length_deferred=True, + ) + uploader.upload() + + if not uploader.url: + fail("deferred-length TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "deferred-length upload accepted {} bytes, expected {}".format( + uploader.offset, + len(content), + ) + ) + if uploader.file_size != len(content): + fail( + "deferred-length upload declared final size {}, expected {}".format( + uploader.file_size, + len(content), + ) + ) + + return { + "acceptedBytes": uploader.offset, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_deferred_length(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} deferred length for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-detailed-error/main.py b/examples/api2-devdock-tus-detailed-error/main.py new file mode 100644 index 0000000..1cc0627 --- /dev/null +++ b/examples/api2-devdock-tus-detailed-error/main.py @@ -0,0 +1,149 @@ +"""Validate detailed TUS create-upload errors from contract scenarios.""" + +import sys +from io import BytesIO +from pathlib import Path + +import requests +from tusclient import client as tus +from tusclient.exceptions import TusDetailedError + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + conformance_input_options, + conformance_input_source_bytes, + fail, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +def conformance_request(conformance_scenario): + requests_list = conformance_scenario["requests"] + if not isinstance(requests_list, list) or len(requests_list) != 1: + fail("detailed error scenario must have exactly one request") + return object_value(requests_list[0], "conformanceScenario.requests[0]") + + +def assert_expected_headers(actual_headers, expected_headers): + normalized_actual_headers = {} + for key, value in actual_headers.items(): + normalized_actual_headers[key.lower()] = value + + for key, expected_value in expected_headers.items(): + actual_value = normalized_actual_headers.get(key.lower()) + if actual_value != expected_value: + fail( + "detailed error expected header {}={!r}, got {!r}".format( + key, + expected_value, + actual_value, + ) + ) + + +def response_for(request_plan): + response_plan = object_value( + request_plan["response"], + "conformanceScenario.requests[0].response", + ) + response = requests.Response() + response.status_code = response_plan["statusCode"] + response._content = string_value( + response_plan["body"], + "conformanceScenario.requests[0].response.body", + ).encode("utf-8") + response.headers.update(response_plan.get("headers") or {}) + return response + + +def upload_expect_detailed_error(conformance_scenario): + request_plan = conformance_request(conformance_scenario) + input_options = conformance_input_options(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + metadata = object_value(input_options["metadata"], "metadata") + headers = object_value(input_options["headers"], "headers") + content = conformance_input_source_bytes(conformance_scenario) + request_methods = [] + request_urls = [] + original_post = requests.post + + def fake_post(url, **kwargs): + request_methods.append(request_plan["effectiveMethod"]) + request_urls.append(url) + if url != request_plan["expectedUrl"]: + fail( + "detailed error expected URL {!r}, got {!r}".format( + request_plan["expectedUrl"], + url, + ) + ) + assert_expected_headers(kwargs["headers"], request_plan["headers"]) + error_message = request_plan.get("errorMessage") + if error_message is not None: + raise requests.exceptions.ConnectionError(error_message) + return response_for(request_plan) + + requests.post = fake_post + try: + try: + tus.TusClient(endpoint_url, headers=headers).uploader( + file_stream=BytesIO(content), + metadata=metadata, + ).create_url() + except Exception as error: + return detailed_result(error, request_methods, request_urls) + finally: + requests.post = original_post + + fail("detailed error scenario unexpectedly created an upload") + + +def detailed_result(error, request_methods, request_urls): + result = { + "errorCaught": True, + "errorIsDetailed": isinstance(error, TusDetailedError), + "errorMessage": str(error), + "requestCount": len(request_methods), + "requestMethods": request_methods, + "requestUrls": request_urls, + } + + if not isinstance(error, TusDetailedError): + return result + + result["causingErrorPresent"] = error.causing_error is not None + if error.causing_error is not None: + result["causingErrorMessage"] = str(error.causing_error) + result["originalRequestMethod"] = error.original_request_method + result["originalRequestRequestId"] = error.original_request_id + result["originalRequestUrl"] = error.original_request_url + result["originalResponsePresent"] = error.original_response_present + if error.original_response_present: + result["originalResponseBody"] = error.original_response_body + result["originalResponseStatus"] = error.original_response_status + + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_expect_detailed_error(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} observed detailed error {}".format( + scenario_id(scenario), + result["errorMessage"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-node-path-input-source/main.py b/examples/api2-devdock-tus-node-path-input-source/main.py new file mode 100644 index 0000000..0d7681a --- /dev/null +++ b/examples/api2-devdock-tus-node-path-input-source/main.py @@ -0,0 +1,114 @@ +"""Read a path-backed source as a TUS upload input.""" + +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + conformance_input_options, + conformance_input_source_bytes, + conformance_input_source_kind, + conformance_scenario_wants_event, + fail, + load_scenario, + object_value, + scenario_id, + write_result, +) + + +def append_source_open_event(events, conformance_scenario, input_kind, size): + if not conformance_scenario_wants_event(conformance_scenario, "source-open"): + return events + + return events + [ + { + "inputKind": input_kind, + "kind": "source-open", + "size": size, + } + ] + + +def append_source_close_event(events, conformance_scenario): + if not conformance_scenario_wants_event(conformance_scenario, "source-close"): + return events + + return events + [{"kind": "source-close"}] + + +def append_success_event(events, conformance_scenario): + if not conformance_scenario_wants_event(conformance_scenario, "success"): + return events + + return events + [{"kind": "success"}] + + +def upload_with_node_path_input_source(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + input_kind = conformance_input_source_kind(conformance_scenario) + endpoint_url = input_options["endpointUrl"] + events = [] + + with TemporaryDirectory(prefix="api2-python-tus-node-path-input-source-") as tmp_dir: + input_path = Path(tmp_dir) / "input.txt" + input_path.write_bytes(content) + events = append_source_open_event( + events, + conformance_scenario, + input_kind, + len(content), + ) + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + uploader = tus.TusClient(conformance_server.endpoint_url()).uploader( + chunk_size=len(content), + file_path=str(input_path), + metadata=object_value(input_options["metadata"], "metadata"), + ) + uploader.upload() + events = append_success_event(events, conformance_scenario) + + if not uploader.url: + fail("node-path TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "node-path TUS upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + events = append_source_close_event(events, conformance_scenario) + conformance_server.assert_exhausted() + result = conformance_server.result() + result["events"] = events + result["inputKind"] = input_kind + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_node_path_input_source(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} read {} for {}".format( + scenario_id(scenario), + result["inputKind"], + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-override-patch-method/main.py b/examples/api2-devdock-tus-override-patch-method/main.py new file mode 100644 index 0000000..8a12a9a --- /dev/null +++ b/examples/api2-devdock-tus-override-patch-method/main.py @@ -0,0 +1,76 @@ +"""Upload TUS bytes by tunneling PATCH through an override POST.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + bool_value, + conformance_input_options, + conformance_input_source_bytes, + fail, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +def upload_with_override_patch_method(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + upload_url = string_value(input_options["uploadUrl"], "uploadUrl") + override_patch_method = bool_value( + input_options["overridePatchMethod"], + "overridePatchMethod", + ) + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + uploader = tus.TusClient(conformance_server.endpoint_url()).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + url=conformance_server.local_url(upload_url), + override_patch_method=override_patch_method, + ) + uploader.upload() + + if not uploader.url: + fail("override PATCH method scenario did not expose an upload URL") + if uploader.offset != len(content): + fail( + "override PATCH method upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_override_patch_method(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} overrode PATCH for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-parallel-upload-concat/main.py b/examples/api2-devdock-tus-parallel-upload-concat/main.py new file mode 100644 index 0000000..f7dea4f --- /dev/null +++ b/examples/api2-devdock-tus-parallel-upload-concat/main.py @@ -0,0 +1,107 @@ +"""Prove TUS parallel partial uploads are concatenated into a final upload.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + conformance_input_options, + conformance_input_source_bytes, + fail, + int_value, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +def upload_with_parallel_concat(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + metadata = object_value(input_options["metadata"], "metadata") + metadata_for_partial_uploads = object_value( + input_options["metadataForPartialUploads"], + "metadataForPartialUploads", + ) + parallel_uploads = int_value(input_options["parallelUploads"], "parallelUploads") + completion = object_value( + conformance_scenario["completion"], + "conformanceScenario.completion", + ) + completion_kind = string_value( + completion["kind"], + "conformanceScenario.completion.kind", + ) + events = [] + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + client = tus.TusClient(conformance_server.endpoint_url()) + uploader = client.uploader( + file_stream=BytesIO(content), + metadata=metadata, + metadata_for_partial_uploads=metadata_for_partial_uploads, + parallel_uploads=parallel_uploads, + on_progress=lambda bytes_sent, bytes_total: events.append( + { + "bytesSent": bytes_sent, + "bytesTotal": bytes_total, + "kind": "progress", + }, + ), + on_chunk_complete=lambda chunk_size, bytes_accepted, bytes_total: events.append( + { + "bytesAccepted": bytes_accepted, + "bytesTotal": bytes_total, + "chunkSize": chunk_size, + "kind": "chunk-complete", + }, + ), + ) + uploader.upload() + + if not uploader.url: + fail("parallel upload concat scenario did not expose an upload URL") + if uploader.offset != len(content): + fail( + "parallel upload concat scenario upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["completionKind"] = completion_kind + result["errorCalled"] = False + result["eventCount"] = len(events) + result["events"] = events + result["successCalled"] = True + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_parallel_concat(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} concatenated parallel uploads into {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-protocol-version-selection/main.py b/examples/api2-devdock-tus-protocol-version-selection/main.py new file mode 100644 index 0000000..d15c5c7 --- /dev/null +++ b/examples/api2-devdock-tus-protocol-version-selection/main.py @@ -0,0 +1,84 @@ +"""Select a generated TUS protocol mode for a conformance upload.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + bool_value, + conformance_input_options, + conformance_input_source_bytes, + fail, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +def upload_with_protocol_version_selection(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + metadata = object_value(input_options["metadata"], "metadata") + protocol = string_value(input_options["protocol"], "protocol") + upload_data_during_creation = bool_value( + input_options["uploadDataDuringCreation"], + "uploadDataDuringCreation", + ) + if not upload_data_during_creation: + fail("Python protocol-version proof expects creation with upload") + + completion = object_value(conformance_scenario["completion"], "completion") + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + uploader = tus.TusClient(conformance_server.endpoint_url()).create_upload_with_data( + len(content), + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=metadata, + protocol=protocol, + ) + + if not uploader.url: + fail("protocol-version scenario did not expose an upload URL") + if uploader.offset != len(content): + fail( + "protocol-version upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["completionKind"] = string_value(completion["kind"], "completion.kind") + result["errorCalled"] = False + result["successCalled"] = True + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_protocol_version_selection(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} selected protocol for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-relative-location-resolution/main.py b/examples/api2-devdock-tus-relative-location-resolution/main.py new file mode 100644 index 0000000..5e334c2 --- /dev/null +++ b/examples/api2-devdock-tus-relative-location-resolution/main.py @@ -0,0 +1,68 @@ +"""Resolve a relative TUS Location header against the creation endpoint.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + conformance_input_options, + conformance_input_source_bytes, + fail, + load_scenario, + object_value, + scenario_id, + write_result, +) + + +def upload_with_relative_location_resolution(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = input_options["endpointUrl"] + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + uploader = tus.TusClient(conformance_server.endpoint_url()).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=object_value(input_options["metadata"], "metadata"), + ) + uploader.upload() + + if not uploader.url: + fail("relative Location scenario did not expose an upload URL") + if uploader.offset != len(content): + fail( + "relative Location upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_relative_location_resolution(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} resolved {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-request-id-headers/main.py b/examples/api2-devdock-tus-request-id-headers/main.py new file mode 100644 index 0000000..58b2f43 --- /dev/null +++ b/examples/api2-devdock-tus-request-id-headers/main.py @@ -0,0 +1,110 @@ +"""Send Transloadit devdock TUS request ID headers.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + scenario_bytes, + scenario_id, + tus_url, + upload_add_request_id, + upload_headers, + upload_metadata, + upload_request_id_header_name, + write_result, +) + + +def record_request_id_header(context, request_id_header_name, custom_request_id): + actual_value = context.headers.get(request_id_header_name) + if actual_value is None: + fail("request ID header {} was not sent".format(request_id_header_name)) + if actual_value == custom_request_id: + fail( + "request ID header {} was not generated; saw custom value {!r}".format( + request_id_header_name, + custom_request_id, + ) + ) + if len(actual_value) != 36 or "-" not in actual_value: + fail( + "request ID header {} expected generated UUID shape, got {!r}".format( + request_id_header_name, + actual_value, + ) + ) + return {request_id_header_name: actual_value} + + +def upload_with_request_id_headers(scenario, create_response): + upload_config = scenario["upload"] + request_id_header_name = upload_request_id_header_name(scenario) + custom_headers = upload_headers(scenario) + custom_request_id = custom_headers[request_id_header_name] + content = scenario_bytes(upload_config) + headers_by_method = {} + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + def before_request(context): + if context.method in ("POST", "PATCH"): + headers_by_method[context.method] = record_request_id_header( + context, + request_id_header_name, + custom_request_id, + ) + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response), + headers=custom_headers, + request_hooks=RequestLifecycleHooks(before_request=before_request), + add_request_id=upload_add_request_id(scenario), + ).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("request ID headers upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "request ID headers upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + for method in ("POST", "PATCH"): + if method not in headers_by_method: + fail("request ID headers did not observe {} request".format(method)) + + return { + "headersByMethod": headers_by_method, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_request_id_headers(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} sent request ID headers for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-request-lifecycle-hooks/main.py b/examples/api2-devdock-tus-request-lifecycle-hooks/main.py new file mode 100644 index 0000000..6e135d7 --- /dev/null +++ b/examples/api2-devdock-tus-request-lifecycle-hooks/main.py @@ -0,0 +1,107 @@ +"""Observe Transloadit devdock TUS request lifecycle hooks.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + request_lifecycle_hooks, + scenario_bytes, + scenario_id, + tus_url, + upload_metadata, + write_result, +) + + +def assert_equal(actual, expected, label): + if actual != expected: + fail("request lifecycle hooks expected {} {}, got {}".format(label, expected, actual)) + + +def upload_with_request_lifecycle_hooks(scenario, create_response): + upload_config = scenario["upload"] + hooks = request_lifecycle_hooks(scenario) + content = scenario_bytes(upload_config) + before_request_methods = [] + after_response_methods = [] + after_response_status_codes = [] + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + def before_request(context): + before_request_methods.append(context.method) + + def after_response(context, response): + after_response_methods.append(context.method) + after_response_status_codes.append(response.status_code) + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response), + request_hooks=RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ), + ).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("request lifecycle hooks upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "request lifecycle hooks upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + assert_equal( + before_request_methods, + hooks["expectedBeforeRequestMethods"], + "before request methods", + ) + assert_equal( + after_response_methods, + hooks["expectedAfterResponseMethods"], + "after response methods", + ) + assert_equal( + after_response_status_codes, + hooks["expectedAfterResponseStatusCodes"], + "after response status codes", + ) + + return { + "afterResponseMethods": after_response_methods, + "afterResponseStatusCodes": after_response_status_codes, + "beforeRequestMethods": before_request_methods, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_request_lifecycle_hooks(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} observed lifecycle hooks for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-resume-upload/main.py b/examples/api2-devdock-tus-resume-upload/main.py new file mode 100644 index 0000000..a48d659 --- /dev/null +++ b/examples/api2-devdock-tus-resume-upload/main.py @@ -0,0 +1,187 @@ +"""Resume a Transloadit devdock TUS upload using tus-py-client.""" + +import sys +from contextlib import contextmanager +from io import BytesIO +from os import remove +from pathlib import Path +from tempfile import NamedTemporaryFile + +from tusclient import client as tus +from tusclient.fingerprint.interface import Fingerprint +from tusclient.storage import filestorage +from tusclient.storage.interface import Storage + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import fail, load_scenario, scenario_bytes, tus_url, upload_metadata, write_result + + +class StaticFingerprint(Fingerprint): + def __init__(self, value): + self.value = value + + def get_fingerprint(self, fs): + return self.value + + +class MemoryStorage(Storage): + def __init__(self): + self.urls = {} + + def get_item(self, key): + return self.urls.get(key) + + def set_item(self, key, value): + self.urls[key] = value + + def remove_item(self, key): + self.urls.pop(key, None) + + def count(self): + return len(self.urls) + + def keys(self): + return list(self.urls.keys()) + + +def uploader_for(scenario, create_response, content, storage): + upload_config = scenario["upload"] + resume = upload_config["resume"] + return tus.TusClient(tus_url(upload_config, scenario, create_response)).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + store_url=True, + remove_fingerprint_on_success=resume["removeFingerprintOnSuccess"], + url_storage=storage, + fingerprinter=StaticFingerprint(resume["fingerprint"]), + retries=upload_config["retries"], + ) + + +def upload_first_chunk_and_pause(scenario, create_response, content, storage): + upload_config = scenario["upload"] + resume = upload_config["resume"] + chunk_size = upload_config["chunkSize"] + if chunk_size["kind"] != "fixed-bytes": + fail("unsupported chunk size policy {!r}".format(chunk_size["kind"])) + + uploader = uploader_for(scenario, create_response, content, storage) + uploader.chunk_size = chunk_size["bytes"] + uploader.upload(stop_at=resume["stopAfterAcceptedBytes"]) + + if uploader.offset != resume["stopAfterAcceptedBytes"]: + fail( + "first upload accepted {}, expected {}".format( + uploader.offset, + resume["stopAfterAcceptedBytes"], + ) + ) + if not uploader.url: + fail("first TUS upload did not expose an upload URL") + + return uploader.url + + +def resume_stored_upload(scenario, create_response, content, storage): + upload_config = scenario["upload"] + uploader = uploader_for(scenario, create_response, content, storage) + uploader.chunk_size = len(content) + uploader.upload() + + if not uploader.url: + fail("resumed TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail("resumed TUS upload offset {}, expected {}".format(uploader.offset, len(content))) + + return uploader.url + + +@contextmanager +def url_storage_for(upload_config): + backend = upload_config.get("urlStorageBackend") + if backend is None: + yield MemoryStorage() + return + + if backend["kind"] != "file": + fail("unsupported URL storage backend {!r}".format(backend["kind"])) + + temp_fp = NamedTemporaryFile(delete=False) + temp_fp.close() + storage = filestorage.FileStorage(temp_fp.name) + try: + yield storage + finally: + storage.close() + remove(temp_fp.name) + + +def upload_with_stored_resume(scenario, create_response): + upload_config = scenario["upload"] + resume = upload_config["resume"] + content = scenario_bytes(upload_config) + + with url_storage_for(upload_config) as storage: + first_upload_url = upload_first_chunk_and_pause(scenario, create_response, content, storage) + previous_upload_count = storage.count() + storage_keys_after_first_upload = storage.keys() + if previous_upload_count != resume["expectedPreviousUploadCount"]: + fail( + "stored upload count {}, expected {}".format( + previous_upload_count, + resume["expectedPreviousUploadCount"], + ) + ) + + upload_url = resume_stored_upload(scenario, create_response, content, storage) + if upload_url != first_upload_url: + fail("resumed upload URL {}, expected {}".format(upload_url, first_upload_url)) + + remaining_previous_upload_count = storage.count() + if remaining_previous_upload_count != resume["expectedRemainingPreviousUploadCount"]: + fail( + "remaining stored upload count {}, expected {}".format( + remaining_previous_upload_count, + resume["expectedRemainingPreviousUploadCount"], + ) + ) + + result = { + "firstUploadUrl": first_upload_url, + "previousUploadCount": previous_upload_count, + "remainingPreviousUploadCount": remaining_previous_upload_count, + "uploadUrl": upload_url, + } + + backend = upload_config.get("urlStorageBackend") + if backend is not None: + expected_key_prefix = backend["expectedStoredUploadKeyPrefix"] + result.update( + { + "storageFileEntryCount": storage.count(), + "storedUploadKeyPrefixMatched": any( + key.startswith(expected_key_prefix) for key in storage_keys_after_first_upload + ), + "urlStorageBackend": backend["kind"], + } + ) + + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_stored_resume(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} resumed {}".format( + scenario["scenarioId"], + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-retry-offset-recovery/main.py b/examples/api2-devdock-tus-retry-offset-recovery/main.py new file mode 100644 index 0000000..8cca2dc --- /dev/null +++ b/examples/api2-devdock-tus-retry-offset-recovery/main.py @@ -0,0 +1,146 @@ +"""Recover a Transloadit devdock TUS upload offset after a retry.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + fixed_chunk_size_bytes, + load_scenario, + retry_offset_recovery, + scenario_bytes, + scenario_id, + tus_url, + upload_metadata, + write_result, +) + + +def int_header(response, header_name): + value = response.headers.get(header_name) + try: + offset = int(value) + except (TypeError, ValueError): + fail( + "retry offset recovery expected numeric {} response header, got {!r}".format( + header_name, + value, + ) + ) + + if offset < 0: + fail("retry offset recovery expected non-negative offset, got {}".format(offset)) + + return offset + + +def assert_request_methods(actual, expected): + if actual != expected: + fail("retry offset recovery expected request methods {}, got {}".format(expected, actual)) + + +def upload_with_retry_offset_recovery(scenario, create_response): + upload_config = scenario["upload"] + retry = retry_offset_recovery(scenario) + content = scenario_bytes(upload_config) + chunk_size = fixed_chunk_size_bytes(scenario) + recovered_offsets = [] + request_methods = [] + failure_candidate_count = 0 + simulated_failure_count = 0 + + def before_request(context): + request_methods.append(context.method) + + def after_response(context, response): + nonlocal failure_candidate_count + nonlocal simulated_failure_count + + if context.method == retry["recoveryResponse"]["method"]: + recovered_offsets.append( + int_header(response, retry["recoveryResponse"]["offsetHeader"]) + ) + + if context.method != retry["failAfterResponse"]["method"]: + return + + failure_candidate_count += 1 + if failure_candidate_count != retry["failAfterResponse"]["occurrence"]: + return + + simulated_failure_count += 1 + response.status_code = 500 + response._content = retry["failAfterResponse"]["message"].encode("utf-8") + + client = tus.TusClient( + tus_url(upload_config, scenario, create_response), + request_hooks=RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ), + ) + uploader = client.uploader( + file_stream=BytesIO(content), + chunk_size=chunk_size, + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + retry_delay=0, + ) + uploader.upload() + + if not uploader.url: + fail("retry offset recovery TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail("retry offset recovery upload offset {}, expected {}".format(uploader.offset, len(content))) + if simulated_failure_count != retry["expectedFailureCount"]: + fail( + "retry offset recovery expected {} simulated failure(s), got {}".format( + retry["expectedFailureCount"], + simulated_failure_count, + ) + ) + if len(recovered_offsets) != retry["expectedRecoveryRequestCount"]: + fail( + "retry offset recovery expected {} recovery request(s), got {}".format( + retry["expectedRecoveryRequestCount"], + len(recovered_offsets), + ) + ) + if recovered_offsets[0] != retry["expectedRecoveredOffset"]: + fail( + "retry offset recovery expected recovered offset {}, got {}".format( + retry["expectedRecoveredOffset"], + recovered_offsets[0], + ) + ) + assert_request_methods(request_methods, retry["expectedRequestMethods"]) + + return { + "recoveredOffsets": recovered_offsets, + "recoveryRequestCount": len(recovered_offsets), + "requestMethods": request_methods, + "simulatedFailureCount": simulated_failure_count, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_retry_offset_recovery(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} recovered offset for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-retry-state-transitions/main.py b/examples/api2-devdock-tus-retry-state-transitions/main.py new file mode 100644 index 0000000..d3d8942 --- /dev/null +++ b/examples/api2-devdock-tus-retry-state-transitions/main.py @@ -0,0 +1,152 @@ +"""Prove TUS retry attempt state resets after recovered progress.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + TusConformancePlanServer, + conformance_input_options, + conformance_input_source_bytes, + conformance_retry_decisions, + fail, + int_array_value, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +class RetryStateObserver: + def __init__(self, retry_decisions, retry_delays): + self.retry_decisions = retry_decisions + self.retry_delays = retry_delays + self.events = [] + self.index = 0 + + def on_should_retry(self, error, retry_attempt): + if self.index >= len(self.retry_decisions): + fail( + "retry state scenario observed unexpected retry attempt {}".format( + retry_attempt, + ) + ) + + expected = self.retry_decisions[self.index] + if retry_attempt != expected["retryAttempt"]: + fail( + "retry state scenario expected retry attempt {}, got {}".format( + expected["retryAttempt"], + retry_attempt, + ) + ) + + decision = expected["decision"] + self.events.append( + { + "decision": decision, + "kind": "should-retry", + "retryAttempt": retry_attempt, + } + ) + + if decision: + if retry_attempt >= len(self.retry_delays): + fail( + "retry state scenario has no delay for retry attempt {}".format( + retry_attempt, + ) + ) + self.events.append( + { + "delay": self.retry_delays[retry_attempt], + "kind": "retry-schedule", + } + ) + + self.index += 1 + return decision + + def assert_complete(self): + if self.index == len(self.retry_decisions): + return + + fail( + "retry state scenario observed {} retry decision(s), expected {}".format( + self.index, + len(self.retry_decisions), + ) + ) + + +def upload_with_retry_state_transitions(conformance_scenario): + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + endpoint_url = string_value(input_options["endpointUrl"], "endpointUrl") + metadata = object_value(input_options["metadata"], "metadata") + retry_delays = int_array_value(input_options["retryDelays"], "retryDelays") + retry_decisions = conformance_retry_decisions(conformance_scenario) + completion = object_value( + conformance_scenario["completion"], + "conformanceScenario.completion", + ) + completion_kind = string_value( + completion["kind"], + "conformanceScenario.completion.kind", + ) + observer = RetryStateObserver(retry_decisions, retry_delays) + + with TusConformancePlanServer(conformance_scenario, endpoint_url) as conformance_server: + client = tus.TusClient(conformance_server.endpoint_url()) + uploader = client.uploader( + file_stream=BytesIO(content), + metadata=metadata, + on_should_retry=observer.on_should_retry, + retry_delays=retry_delays, + ) + uploader.upload() + observer.assert_complete() + + if not uploader.url: + fail("retry state scenario did not expose an upload URL") + if uploader.offset != len(content): + fail( + "retry state scenario upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + + conformance_server.assert_exhausted() + result = conformance_server.result() + result["completionKind"] = completion_kind + result["errorCalled"] = False + result["eventCount"] = len(observer.events) + result["events"] = observer.events + result["successCalled"] = True + result["uploadUrl"] = conformance_server.canonical_url(uploader.url) + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + result = upload_with_retry_state_transitions(conformance_scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} proved retry state transitions".format( + scenario_id(scenario), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-start-option-validation/main.py b/examples/api2-devdock-tus-start-option-validation/main.py new file mode 100644 index 0000000..b644391 --- /dev/null +++ b/examples/api2-devdock-tus-start-option-validation/main.py @@ -0,0 +1,122 @@ +"""Validate conflicting TUS start options before transport.""" + +import sys +from io import BytesIO +from pathlib import Path + +import requests +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + conformance_input_options, + conformance_input_source_bytes, + fail, + int_value, + load_scenario, + object_value, + scenario_id, + string_value, + write_result, +) + + +def bool_value(value, label): + if not isinstance(value, bool): + fail("{} must be a boolean".format(label)) + return value + + +def conformance_completion(conformance_scenario): + return object_value( + conformance_scenario["completion"], + "conformanceScenario.completion", + ) + + +def run_with_request_counter(operation): + request_count = 0 + original_head = requests.head + original_patch = requests.patch + original_post = requests.post + original_request = requests.request + + def record_request(*args, **kwargs): + nonlocal request_count + request_count += 1 + fail("start option validation made an unexpected HTTP request") + + requests.head = record_request + requests.patch = record_request + requests.post = record_request + requests.request = record_request + try: + return operation(), request_count + finally: + requests.head = original_head + requests.patch = original_patch + requests.post = original_post + requests.request = original_request + + +def validate_start_options(scenario): + conformance_scenario = object_value( + scenario["conformanceScenario"], + "conformanceScenario", + ) + completion = conformance_completion(conformance_scenario) + input_options = conformance_input_options(conformance_scenario) + content = conformance_input_source_bytes(conformance_scenario) + expected_message = string_value( + completion["message"], + "conformanceScenario.completion.message", + ) + + def start_upload(): + try: + tus.TusClient(input_options["endpointUrl"]).uploader( + file_stream=BytesIO(content), + parallel_uploads=int_value( + input_options["parallelUploads"], + "conformanceScenario.inputOptionEntries.parallelUploads", + ), + upload_data_during_creation=bool_value( + input_options.get("uploadDataDuringCreation", False), + "conformanceScenario.inputOptionEntries.uploadDataDuringCreation", + ), + url=input_options.get("uploadUrl"), + ) + except ValueError as error: + return { + "errorCaught": True, + "errorMessage": str(error), + } + + fail("start option validation unexpectedly accepted conflicting options") + + result, request_count = run_with_request_counter(start_upload) + result["requestCount"] = request_count + if result["errorMessage"] != expected_message: + fail( + "start option validation expected error {!r}, got {!r}".format( + expected_message, + result["errorMessage"], + ) + ) + + return result + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + result = validate_start_options(scenario) + write_result(result) + print( + "Python TUS SDK devdock scenario {} rejected conflicting start options".format( + scenario_id(scenario), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-terminate-upload/main.py b/examples/api2-devdock-tus-terminate-upload/main.py new file mode 100644 index 0000000..bb99508 --- /dev/null +++ b/examples/api2-devdock-tus-terminate-upload/main.py @@ -0,0 +1,116 @@ +"""Terminate a Transloadit devdock TUS upload.""" + +import sys +from io import BytesIO +from pathlib import Path + +import requests +from tusclient import client as tus +from tusclient.protocol_generated import DEFAULT_REQUEST_HEADERS +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + fixed_chunk_size_bytes, + load_scenario, + scenario_bytes, + scenario_id, + termination, + tus_url, + upload_metadata, + write_result, +) + + +def count_method(methods, expected_method): + count = 0 + for method in methods: + if method == expected_method: + count += 1 + return count + + +def verify_terminated_upload(termination_plan, upload_url): + response = requests.request( + termination_plan["verificationMethod"], + upload_url, + headers=DEFAULT_REQUEST_HEADERS, + ) + return response.status_code + + +def upload_and_terminate(scenario, create_response): + upload_config = scenario["upload"] + termination_plan = termination(scenario) + content = scenario_bytes(upload_config) + chunk_size = fixed_chunk_size_bytes(scenario) + request_methods = [] + + if termination_plan["stopAfterAcceptedBytes"] > len(content): + fail( + "terminate upload stop-after bytes {} exceeds content length {}".format( + termination_plan["stopAfterAcceptedBytes"], + len(content), + ) + ) + + def before_request(context): + request_methods.append(context.method) + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response), + request_hooks=RequestLifecycleHooks(before_request=before_request), + ).uploader( + file_stream=BytesIO(content), + chunk_size=chunk_size, + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload(stop_at=termination_plan["stopAfterAcceptedBytes"]) + + if not uploader.url: + fail("terminate upload did not expose an upload URL") + if uploader.offset != termination_plan["stopAfterAcceptedBytes"]: + fail( + "terminate upload accepted {} bytes, expected {}".format( + uploader.offset, + termination_plan["stopAfterAcceptedBytes"], + ) + ) + + uploader.client.terminate_upload(uploader.url) + verification_status = verify_terminated_upload(termination_plan, uploader.url) + if verification_status != termination_plan["expectedVerificationStatus"]: + fail( + "terminate upload verification status {}, expected {}".format( + verification_status, + termination_plan["expectedVerificationStatus"], + ) + ) + + return { + "acceptedBytes": uploader.offset, + "deleteRequestCount": count_method(request_methods, termination_plan["method"]), + "requestMethods": request_methods, + "terminated": True, + "uploadUrl": uploader.url, + "verificationStatus": verification_status, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_and_terminate(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} terminated {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-upload-body-headers/main.py b/examples/api2-devdock-tus-upload-body-headers/main.py new file mode 100644 index 0000000..b20c435 --- /dev/null +++ b/examples/api2-devdock-tus-upload-body-headers/main.py @@ -0,0 +1,99 @@ +"""Observe Transloadit devdock TUS upload body headers.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus +from tusclient.request_lifecycle import RequestLifecycleHooks + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + scenario_bytes, + scenario_id, + tus_url, + upload_body_headers_by_method, + upload_metadata, + write_result, +) + + +def record_body_headers(context, expected_headers): + observed = {} + for header_name, expected_value in expected_headers.items(): + actual_value = context.headers.get(header_name) + if actual_value != expected_value: + fail( + "upload body header {} for {} expected {!r}, got {!r}".format( + header_name, + context.method, + expected_value, + actual_value, + ) + ) + observed[header_name] = actual_value + return observed + + +def upload_with_body_headers(scenario, create_response): + upload_config = scenario["upload"] + expected_headers_by_method = upload_body_headers_by_method(scenario) + content = scenario_bytes(upload_config) + body_headers_by_method = {} + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + def before_request(context): + if context.method in expected_headers_by_method: + body_headers_by_method[context.method] = record_body_headers( + context, + expected_headers_by_method[context.method], + ) + + uploader = tus.TusClient( + tus_url(upload_config, scenario, create_response), + request_hooks=RequestLifecycleHooks(before_request=before_request), + ).uploader( + file_stream=BytesIO(content), + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + ) + uploader.upload() + + if not uploader.url: + fail("upload body headers TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail( + "upload body headers upload offset {}, expected {}".format( + uploader.offset, + len(content), + ) + ) + for method in expected_headers_by_method: + if method not in body_headers_by_method: + fail("upload body headers did not observe {} request".format(method)) + + return { + "bodyHeadersByMethod": body_headers_by_method, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_body_headers(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} observed upload body headers for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2-devdock-tus-upload-callbacks/main.py b/examples/api2-devdock-tus-upload-callbacks/main.py new file mode 100644 index 0000000..48132a1 --- /dev/null +++ b/examples/api2-devdock-tus-upload-callbacks/main.py @@ -0,0 +1,122 @@ +"""Observe Transloadit devdock TUS upload callbacks.""" + +import sys +from io import BytesIO +from pathlib import Path + +from tusclient import client as tus + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from api2devdock import ( + fail, + load_scenario, + match_upload_callback_event_keys, + scenario_bytes, + scenario_id, + tus_url, + upload_callback_event_key, + upload_callback_event_key_number, + upload_callback_event_key_total, + upload_callbacks, + upload_metadata, + write_result, +) + + +class EventRecordingBytesIO(BytesIO): + def __init__(self, content, callbacks, events): + super().__init__(content) + self.callbacks = callbacks + self.events = events + + def close(self): + self.events.append( + upload_callback_event_key( + self.callbacks, + self.callbacks["eventKinds"]["sourceClose"], + ) + ) + super().close() + + +def upload_with_callbacks(scenario, create_response): + upload_config = scenario["upload"] + callbacks = upload_callbacks(scenario) + content = scenario_bytes(upload_config) + events = [] + if upload_config["chunkSize"] != "full-file": + fail("unsupported chunk size policy {!r}".format(upload_config["chunkSize"])) + + source = EventRecordingBytesIO(content, callbacks, events) + + def on_progress(bytes_sent, bytes_total): + events.append( + upload_callback_event_key( + callbacks, + callbacks["eventKinds"]["progress"], + upload_callback_event_key_number(bytes_sent), + upload_callback_event_key_total(bytes_total), + ) + ) + + def on_chunk_complete(chunk_size, bytes_accepted, bytes_total): + events.append( + upload_callback_event_key( + callbacks, + callbacks["eventKinds"]["chunkComplete"], + upload_callback_event_key_number(chunk_size), + upload_callback_event_key_number(bytes_accepted), + upload_callback_event_key_total(bytes_total), + ) + ) + + uploader = tus.TusClient(tus_url(upload_config, scenario, create_response)).uploader( + file_stream=source, + chunk_size=len(content), + metadata=upload_metadata(upload_config, scenario, create_response), + retries=upload_config["retries"], + on_progress=on_progress, + on_chunk_complete=on_chunk_complete, + ) + uploader.set_url(uploader.create_url()) + uploader.offset = 0 + events.append( + upload_callback_event_key( + callbacks, + callbacks["eventKinds"]["uploadUrlAvailable"], + ) + ) + + uploader.upload() + + if not uploader.url: + fail("upload callbacks TUS upload did not expose an upload URL") + if uploader.offset != len(content): + fail("upload callbacks upload offset {}, expected {}".format(uploader.offset, len(content))) + + events.append(upload_callback_event_key(callbacks, callbacks["eventKinds"]["success"])) + source.close() + matched_events = match_upload_callback_event_keys(callbacks, events) + + return { + "eventKeys": matched_events, + "rawEventKeys": events, + "uploadUrl": uploader.url, + } + + +def main(): + scenario = load_scenario(Path(__file__).with_name("api2-scenario.json")) + create_response = scenario["prepared"]["createResponse"] + result = upload_with_callbacks(scenario, create_response) + write_result(result) + print( + "Python TUS SDK devdock scenario {} observed upload callbacks for {}".format( + scenario_id(scenario), + result["uploadUrl"], + ) + ) + + +if __name__ == "__main__": + main() diff --git a/examples/api2devdock.py b/examples/api2devdock.py new file mode 100644 index 0000000..1eea1ca --- /dev/null +++ b/examples/api2devdock.py @@ -0,0 +1,849 @@ +"""Shared helpers for API2 devdock examples.""" + +from http.server import BaseHTTPRequestHandler, HTTPServer +import json +import os +from pathlib import Path +from threading import Thread +from urllib.parse import urlparse, urlunparse + + +def fail(message): + raise RuntimeError(message) + + +def load_scenario(default_path): + configured_path = os.environ.get("API2_SDK_EXAMPLE_SCENARIO") + scenario_path = Path(configured_path) if configured_path else Path(default_path) + with scenario_path.open(encoding="utf-8") as scenario_file: + return json.load(scenario_file) + + +def read_path(value, path_parts, label): + current = value + for part in path_parts: + if isinstance(current, list) and isinstance(part, int): + if part >= len(current): + fail("{} path {!r} index {} is out of range".format(label, path_parts, part)) + current = current[part] + continue + + if isinstance(current, dict) and isinstance(part, str): + if part not in current: + fail("{} path {!r} is missing key {!r}".format(label, path_parts, part)) + current = current[part] + continue + + fail("{} path {!r} cannot read {!r} from {!r}".format(label, path_parts, part, current)) + + return current + + +def object_value(value, label): + if not isinstance(value, dict): + fail("{} must be an object".format(label)) + return value + + +def string_value(value, label): + if not isinstance(value, str): + fail("{} must be a string".format(label)) + return value + + +def int_value(value, label): + if not isinstance(value, int) or isinstance(value, bool): + fail("{} must be an integer".format(label)) + return value + + +def bool_value(value, label): + if not isinstance(value, bool): + fail("{} must be a boolean".format(label)) + return value + + +def string_array_value(value, label): + if not isinstance(value, list): + fail("{} must be a list".format(label)) + for index, item in enumerate(value): + string_value(item, "{}[{}]".format(label, index)) + return value + + +def int_array_value(value, label): + if not isinstance(value, list): + fail("{} must be a list".format(label)) + for index, item in enumerate(value): + int_value(item, "{}[{}]".format(label, index)) + return value + + +def string_array_array_value(value, label): + if not isinstance(value, list): + fail("{} must be a list".format(label)) + for index, item in enumerate(value): + string_array_value(item, "{}[{}]".format(label, index)) + return value + + +def string_map_value(value, label): + if not isinstance(value, dict): + fail("{} must be an object".format(label)) + for key, item in value.items(): + string_value(key, "{} key".format(label)) + string_value(item, "{}.{}".format(label, key)) + return value + + +def conformance_input_options(conformance_scenario): + entries = conformance_scenario["inputOptionEntries"] + if not isinstance(entries, list): + fail("conformanceScenario.inputOptionEntries must be a list") + + result = {} + for index, entry in enumerate(entries): + option = object_value( + entry, + "conformanceScenario.inputOptionEntries[{}]".format(index), + ) + key = string_value( + option["key"], + "conformanceScenario.inputOptionEntries[{}].key".format(index), + ) + result[key] = option["value"] + + return result + + +def conformance_input_source_bytes(conformance_scenario): + input_source = object_value( + conformance_scenario["inputSource"], + "conformanceScenario.inputSource", + ) + kind = string_value(input_source["kind"], "conformanceScenario.inputSource.kind") + if kind not in ("blob", "node-path-reference"): + fail("unsupported conformance input source kind {!r}".format(kind)) + + return string_value( + input_source["content"], + "conformanceScenario.inputSource.content", + ).encode("utf-8") + + +def conformance_input_source_kind(conformance_scenario): + input_source = object_value( + conformance_scenario["inputSource"], + "conformanceScenario.inputSource", + ) + return string_value(input_source["kind"], "conformanceScenario.inputSource.kind") + + +def conformance_retry_decisions(conformance_scenario): + decisions = conformance_scenario["retryDecisions"] + if not isinstance(decisions, list): + fail("conformanceScenario.retryDecisions must be a list") + + result = [] + for index, decision in enumerate(decisions): + decision = object_value( + decision, + "conformanceScenario.retryDecisions[{}]".format(index), + ) + result.append( + { + "decision": bool_value( + decision["decision"], + "conformanceScenario.retryDecisions[{}].decision".format(index), + ), + "retryAttempt": int_value( + decision["retryAttempt"], + "conformanceScenario.retryDecisions[{}].retryAttempt".format(index), + ), + } + ) + + return result + + +def conformance_scenario_wants_event(conformance_scenario, event_kind): + events = conformance_scenario.get("events", []) + if not isinstance(events, list): + fail("conformanceScenario.events must be a list") + + for index, event in enumerate(events): + event = object_value(event, "conformanceScenario.events[{}]".format(index)) + kind = string_value( + event["kind"], + "conformanceScenario.events[{}].kind".format(index), + ) + if kind == event_kind: + return True + + return False + + +def resolve_value(value_spec, context, label): + if "value" in value_spec: + return value_spec["value"] + + source = value_spec.get("source") + if not isinstance(source, dict): + fail("{} value spec has no literal value or source".format(label)) + + root = source.get("root") + if root not in context: + fail("{} value source root {!r} is unavailable".format(label, root)) + + path_parts = source.get("path") or [] + if not isinstance(path_parts, list): + fail("{} value source path must be a list".format(label)) + + return read_path(context[root], path_parts, label) + + +def scenario_bytes(upload_config): + source = upload_config["source"] + if source["kind"] != "bytes": + fail("unsupported scenario source kind {!r}".format(source["kind"])) + if source["encoding"] != "utf8": + fail("unsupported scenario source encoding {!r}".format(source["encoding"])) + return source["value"].encode("utf-8") + + +def fixed_chunk_size_bytes(scenario): + upload = object_value(scenario["upload"], "upload") + chunk_size = object_value(upload["chunkSize"], "upload.chunkSize") + kind = string_value(chunk_size["kind"], "upload.chunkSize.kind") + if kind != "fixed-bytes": + fail("unsupported chunk size kind {!r}".format(kind)) + bytes_value = int_value(chunk_size["bytes"], "upload.chunkSize.bytes") + if bytes_value <= 0: + fail("upload.chunkSize.bytes must be positive") + return bytes_value + + +def retry_offset_recovery(scenario): + upload = object_value(scenario["upload"], "upload") + retry = object_value(upload["retryOffsetRecovery"], "upload.retryOffsetRecovery") + fail_after_response = object_value( + retry["failAfterResponse"], + "upload.retryOffsetRecovery.failAfterResponse", + ) + recovery_response = object_value( + retry["recoveryResponse"], + "upload.retryOffsetRecovery.recoveryResponse", + ) + return { + "expectedFailureCount": int_value( + retry["expectedFailureCount"], + "upload.retryOffsetRecovery.expectedFailureCount", + ), + "expectedRecoveredOffset": int_value( + retry["expectedRecoveredOffset"], + "upload.retryOffsetRecovery.expectedRecoveredOffset", + ), + "expectedRecoveryRequestCount": int_value( + retry["expectedRecoveryRequestCount"], + "upload.retryOffsetRecovery.expectedRecoveryRequestCount", + ), + "expectedRequestMethods": string_array_value( + retry["expectedRequestMethods"], + "upload.retryOffsetRecovery.expectedRequestMethods", + ), + "failAfterResponse": { + "message": string_value( + fail_after_response["message"], + "upload.retryOffsetRecovery.failAfterResponse.message", + ), + "method": string_value( + fail_after_response["method"], + "upload.retryOffsetRecovery.failAfterResponse.method", + ), + "occurrence": int_value( + fail_after_response["occurrence"], + "upload.retryOffsetRecovery.failAfterResponse.occurrence", + ), + }, + "recoveryResponse": { + "method": string_value( + recovery_response["method"], + "upload.retryOffsetRecovery.recoveryResponse.method", + ), + "offsetHeader": string_value( + recovery_response["offsetHeader"], + "upload.retryOffsetRecovery.recoveryResponse.offsetHeader", + ), + }, + } + + +def request_lifecycle_hooks(scenario): + upload = object_value(scenario["upload"], "upload") + hooks = object_value(upload["requestLifecycleHooks"], "upload.requestLifecycleHooks") + return { + "expectedAfterResponseMethods": string_array_value( + hooks["expectedAfterResponseMethods"], + "upload.requestLifecycleHooks.expectedAfterResponseMethods", + ), + "expectedAfterResponseStatusCodes": int_array_value( + hooks["expectedAfterResponseStatusCodes"], + "upload.requestLifecycleHooks.expectedAfterResponseStatusCodes", + ), + "expectedBeforeRequestMethods": string_array_value( + hooks["expectedBeforeRequestMethods"], + "upload.requestLifecycleHooks.expectedBeforeRequestMethods", + ), + } + + +def upload_headers(scenario): + upload = object_value(scenario["upload"], "upload") + return string_map_value(upload["headers"], "upload.headers") + + +def upload_body_headers_by_method(scenario): + upload = object_value(scenario["upload"], "upload") + body_headers_by_method = object_value( + upload["bodyHeadersByMethod"], + "upload.bodyHeadersByMethod", + ) + result = {} + for method, headers in body_headers_by_method.items(): + string_value(method, "upload.bodyHeadersByMethod key") + result[method] = string_map_value( + headers, + "upload.bodyHeadersByMethod.{}".format(method), + ) + return result + + +def upload_add_request_id(scenario): + upload = object_value(scenario["upload"], "upload") + value = upload["addRequestId"] + if not isinstance(value, bool): + fail("upload.addRequestId must be a boolean") + return value + + +def upload_request_id_header_name(scenario): + upload = object_value(scenario["upload"], "upload") + return string_value(upload["requestIdHeaderName"], "upload.requestIdHeaderName") + + +def upload_callbacks(scenario): + upload = object_value(scenario["upload"], "upload") + callbacks = object_value(upload["uploadCallbacks"], "upload.uploadCallbacks") + event_kinds = object_value( + callbacks["eventKinds"], + "upload.uploadCallbacks.eventKinds", + ) + return { + "allowedExtraEventKeyPrefixes": string_array_value( + callbacks["allowedExtraEventKeyPrefixes"], + "upload.uploadCallbacks.allowedExtraEventKeyPrefixes", + ), + "eventKeyAlternativeGroups": string_array_array_value( + callbacks["eventKeyAlternativeGroups"], + "upload.uploadCallbacks.eventKeyAlternativeGroups", + ), + "eventKinds": { + "chunkComplete": string_value( + event_kinds["chunkComplete"], + "upload.uploadCallbacks.eventKinds.chunkComplete", + ), + "progress": string_value( + event_kinds["progress"], + "upload.uploadCallbacks.eventKinds.progress", + ), + "sourceClose": string_value( + event_kinds["sourceClose"], + "upload.uploadCallbacks.eventKinds.sourceClose", + ), + "success": string_value( + event_kinds["success"], + "upload.uploadCallbacks.eventKinds.success", + ), + "uploadUrlAvailable": string_value( + event_kinds["uploadUrlAvailable"], + "upload.uploadCallbacks.eventKinds.uploadUrlAvailable", + ), + }, + "eventKeyPartSeparator": string_value( + callbacks["eventKeyPartSeparator"], + "upload.uploadCallbacks.eventKeyPartSeparator", + ), + "eventKeys": string_array_value( + callbacks["eventKeys"], + "upload.uploadCallbacks.eventKeys", + ), + "eventPolicyMatching": string_value( + callbacks["eventPolicyMatching"], + "upload.uploadCallbacks.eventPolicyMatching", + ), + } + + +def termination(scenario): + upload = object_value(scenario["upload"], "upload") + termination_config = object_value(upload["termination"], "upload.termination") + return { + "expectedVerificationStatus": int_value( + termination_config["expectedVerificationStatus"], + "upload.termination.expectedVerificationStatus", + ), + "method": string_value( + termination_config["method"], + "upload.termination.method", + ), + "minimumDeleteRequestCount": int_value( + termination_config["minimumDeleteRequestCount"], + "upload.termination.minimumDeleteRequestCount", + ), + "stopAfterAcceptedBytes": int_value( + termination_config["stopAfterAcceptedBytes"], + "upload.termination.stopAfterAcceptedBytes", + ), + "verificationMethod": string_value( + termination_config["verificationMethod"], + "upload.termination.verificationMethod", + ), + } + + +def upload_callback_event_key(callbacks, *parts): + return callbacks["eventKeyPartSeparator"].join(parts) + + +def upload_callback_event_key_number(value): + return str(value) + + +def upload_callback_event_key_total(value): + return scalar_string(value) + + +def upload_callback_event_matches_expected(callbacks, expected_index, actual): + if actual == callbacks["eventKeys"][expected_index]: + return True + + if expected_index >= len(callbacks["eventKeyAlternativeGroups"]): + return False + + return actual in callbacks["eventKeyAlternativeGroups"][expected_index] + + +def has_allowed_upload_callback_extra_event_prefix(callbacks, event): + for prefix in callbacks["allowedExtraEventKeyPrefixes"]: + if event.startswith(prefix): + return True + + return False + + +def match_upload_callback_event_keys(callbacks, actual): + policy = callbacks["eventPolicyMatching"] + if policy not in ("exact", "exact-except-allowed-extra-events"): + fail("unsupported upload callback event policy {!r}".format(policy)) + + expected_index = 0 + matched = [] + for event in actual: + if expected_index < len(callbacks["eventKeys"]) and upload_callback_event_matches_expected( + callbacks, + expected_index, + event, + ): + matched.append(callbacks["eventKeys"][expected_index]) + expected_index += 1 + continue + + if policy == "exact-except-allowed-extra-events" and has_allowed_upload_callback_extra_event_prefix( + callbacks, + event, + ): + continue + + fail( + "upload callback events emitted unexpected extra event {!r}; allowed prefixes {}; expected {}, got {}".format( + event, + callbacks["allowedExtraEventKeyPrefixes"], + callbacks["eventKeys"], + actual, + ) + ) + + if expected_index != len(callbacks["eventKeys"]): + fail( + "upload callback events did not emit every expected non-extra event; expected {}, got {}".format( + callbacks["eventKeys"], + actual, + ) + ) + + return matched + + +def scenario_id(scenario): + return string_value(scenario["scenarioId"], "scenarioId") + + +def scalar_string(value): + if value is None: + return "null" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +class TusConformancePlanServer: + def __init__(self, conformance_scenario, endpoint_origin, on_abort_request=None): + self.endpoint_origin = urlparse(string_value(endpoint_origin, "endpointOrigin")) + if not self.endpoint_origin.scheme or not self.endpoint_origin.netloc: + fail("endpointOrigin must be an absolute URL") + + self.on_abort_request = on_abort_request + self.input_source_content = conformance_input_source_bytes(conformance_scenario) + requests = conformance_scenario["requests"] + if not isinstance(requests, list): + fail("conformanceScenario.requests must be a list") + self.requests = requests + self.errors = [] + self.events = [] + self.observed = [None] * len(requests) + self.observed_count = 0 + self.next_request_index = 0 + self.httpd = HTTPServer(("127.0.0.1", 0), self._handler_class()) + self.thread = Thread(target=self.httpd.serve_forever) + self.thread.daemon = True + self.thread.start() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def close(self): + self.httpd.shutdown() + self.httpd.server_close() + self.thread.join(timeout=5) + + def endpoint_url(self): + return self.local_url(urlunparse(self.endpoint_origin)) + + def local_url(self, canonical_url): + parsed = urlparse(canonical_url) + if ( + parsed.scheme != self.endpoint_origin.scheme + or parsed.netloc != self.endpoint_origin.netloc + ): + return canonical_url + + server_origin = self._server_origin() + return urlunparse( + ( + server_origin.scheme, + server_origin.netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ) + ) + + def canonical_url(self, actual_url): + parsed = urlparse(actual_url) + server_origin = self._server_origin() + if parsed.scheme != server_origin.scheme or parsed.netloc != server_origin.netloc: + return actual_url + + return urlunparse( + ( + self.endpoint_origin.scheme, + self.endpoint_origin.netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment, + ) + ) + + def local_value(self, value): + return string_value(value, "value").replace( + self._origin_string(self.endpoint_origin), + self._origin_string(self._server_origin()), + ) + + def canonical_value(self, value): + return string_value(value, "value").replace( + self._origin_string(self._server_origin()), + self._origin_string(self.endpoint_origin), + ) + + def assert_exhausted(self): + self.assert_no_errors() + if self.observed_count == len(self.requests): + return + + fail( + "expected {} conformance request(s), got {}".format( + len(self.requests), + self.observed_count, + ) + ) + + def assert_no_errors(self): + if self.errors: + fail("; ".join(self.errors)) + + def result(self): + self.assert_no_errors() + observed = [request for request in self.observed if request is not None] + return { + "absentHeaderPresence": [ + request["absentHeaderPresence"] for request in observed + ], + "events": self.events, + "requestBodySizes": [request["bodySize"] for request in observed], + "requestBodyStarts": [request["bodyStart"] for request in observed], + "requestCount": self.observed_count, + "requestHeaders": [request["headers"] for request in observed], + "requestMethods": [request["method"] for request in observed], + "requestUrls": [request["url"] for request in observed], + } + + def _handler_class(self): + conformance_server = self + + class TusConformanceRequestHandler(BaseHTTPRequestHandler): + def do_HEAD(self): + self._handle_conformance_request() + + def do_PATCH(self): + self._handle_conformance_request() + + def do_POST(self): + self._handle_conformance_request() + + def do_DELETE(self): + self._handle_conformance_request() + + def log_message(self, format, *args): + return + + def _handle_conformance_request(self): + content_length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(content_length) if content_length > 0 else b"" + try: + request_plan = conformance_server.observe_request(self, body) + if request_plan.get("abort", False): + conformance_server.abort_request(conformance_server.observed_count - 1) + self.close_connection = True + return + conformance_server.write_response(self, request_plan) + except Exception as error: + conformance_server.errors.append(str(error)) + response_body = str(error).encode("utf-8") + self.send_response(500) + self.send_header("Content-Length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + + return TusConformanceRequestHandler + + def observe_request(self, handler, body): + if self.next_request_index >= len(self.requests): + fail("unexpected request {} {}".format(handler.command, handler.path)) + + request_plan = object_value( + self.requests[self.next_request_index], + "conformanceScenario.requests[{}]".format(self.next_request_index), + ) + actual_url = self.canonical_url(self._request_url(handler)) + self.assert_request_matches_plan( + self.next_request_index, + request_plan, + handler.command, + actual_url, + body, + ) + self.assert_request_body_content(self.next_request_index, request_plan, body) + self.assert_absent_headers(self.next_request_index, request_plan, handler.headers) + expected_headers = object_value( + request_plan["effectiveHeaders"], + "conformanceScenario.requests[{}].effectiveHeaders".format( + self.next_request_index, + ), + ) + self.assert_headers(self.next_request_index, expected_headers, handler.headers) + + self.observed[self.next_request_index] = { + "absentHeaderPresence": self.captured_absent_header_presence( + request_plan, + handler.headers, + ), + "bodySize": None if request_plan.get("bodySize") is None else len(body), + "bodyStart": request_plan.get("bodyStart"), + "headers": self.captured_headers(expected_headers, handler.headers), + "method": handler.command, + "url": actual_url, + } + self.observed_count += 1 + self.next_request_index += 1 + return request_plan + + def abort_request(self, request_index): + observed = self.observed[request_index] + event = { + "kind": "request-abort", + "method": observed["method"], + "requestIndex": request_index, + "url": observed["url"], + } + self.events.append(event) + if self.on_abort_request is not None: + self.on_abort_request(event) + + def assert_request_matches_plan(self, request_index, request_plan, method, actual_url, body): + expected_method = string_value( + request_plan["effectiveMethod"], + "conformanceScenario.requests[{}].effectiveMethod".format(request_index), + ) + expected_url = string_value( + request_plan["expectedUrl"], + "conformanceScenario.requests[{}].expectedUrl".format(request_index), + ) + if method != expected_method: + fail( + "request {} expected method {}, got {}".format( + request_index, + expected_method, + method, + ) + ) + if actual_url != expected_url: + fail( + "request {} expected URL {}, got {}".format( + request_index, + expected_url, + actual_url, + ) + ) + body_size = request_plan.get("bodySize") + if body_size is not None and len(body) != body_size: + fail( + "request {} expected body size {}, got {}".format( + request_index, + body_size, + len(body), + ) + ) + + def assert_request_body_content(self, request_index, request_plan, body): + body_start = request_plan.get("bodyStart") + if body_start is None: + return + + expected = self.input_source_content[body_start : body_start + len(body)] + if body != expected: + fail("request {} body did not match input source slice".format(request_index)) + + def assert_absent_headers(self, request_index, request_plan, actual_headers): + normalized_headers = self._normalized_headers(actual_headers) + for name in request_plan["absentHeaders"]: + if name.lower() not in normalized_headers: + continue + + fail("request {} expected header {} to be absent".format(request_index, name)) + + def assert_headers(self, request_index, expected_headers, actual_headers): + normalized_headers = self._normalized_headers(actual_headers) + for name, expected_value in expected_headers.items(): + actual_value = normalized_headers.get(name.lower()) + local_expected_value = self.local_value(expected_value) + if actual_value == local_expected_value: + continue + + fail( + "request {} expected header {}={!r}, got {!r}".format( + request_index, + name, + local_expected_value, + actual_value, + ) + ) + + def captured_headers(self, expected_headers, actual_headers): + normalized_headers = self._normalized_headers(actual_headers) + result = {} + for name in expected_headers: + actual_value = normalized_headers.get(name.lower()) + if actual_value is not None: + result[name] = self.canonical_value(actual_value) + + return result + + def captured_absent_header_presence(self, request_plan, actual_headers): + normalized_headers = self._normalized_headers(actual_headers) + result = {} + for name in request_plan["absentHeaders"]: + result[name] = name.lower() in normalized_headers + return result + + def write_response(self, handler, request_plan): + response_plan = object_value( + request_plan["response"], + "conformanceScenario request response", + ) + response_body = (response_plan.get("body") or "").encode("utf-8") + handler.send_response(response_plan["statusCode"]) + for name, value in response_plan["effectiveHeaders"].items(): + handler.send_header(name, self.local_value(value)) + if response_body: + handler.send_header("Content-Length", str(len(response_body))) + handler.end_headers() + if response_body: + handler.wfile.write(response_body) + + def _server_origin(self): + host, port = self.httpd.server_address + return urlparse("http://{}:{}".format(host, port)) + + def _request_url(self, handler): + host = handler.headers.get("Host") + if host is None: + host = "{}:{}".format(*self.httpd.server_address) + return "http://{}{}".format(host, handler.path) + + def _normalized_headers(self, headers): + return {name.lower(): value for name, value in headers.items()} + + @staticmethod + def _origin_string(parsed_url): + return "{}://{}".format(parsed_url.scheme, parsed_url.netloc) + + +def tus_url(upload_config, scenario, create_response): + context = {"createResponse": create_response, "scenario": scenario} + return scalar_string(resolve_value(upload_config["tusUrl"], context, "tusUrl")) + + +def upload_metadata(upload_config, scenario, create_response): + context = {"createResponse": create_response, "scenario": scenario} + metadata = {} + for field in upload_config["metadata"]: + metadata[field["name"]] = scalar_string( + resolve_value(field["value"], context, field["name"]) + ) + return metadata + + +def write_result(result): + result_path = os.environ.get("API2_SDK_EXAMPLE_RESULT") + if not result_path: + return + + with Path(result_path).open("w", encoding="utf-8") as result_file: + json.dump(result, result_file, indent=2) + result_file.write("\n") diff --git a/setup.py b/setup.py index a48a4e9..c868a76 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ install_requires=[ 'requests>=2.18.4', 'tinydb>=3.5.0', - 'aiohttp>=3.6.2' + 'aiohttp>=3.6.2,<3.14' ], extras_require={ 'test': [ diff --git a/tests/generated_protocol_contract.py b/tests/generated_protocol_contract.py new file mode 100644 index 0000000..fd513f7 --- /dev/null +++ b/tests/generated_protocol_contract.py @@ -0,0 +1,7190 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +TUS_WIRE_VERSIONS = [ + { + 'default': True, + 'value': '1.0.0', + }, +] + +TUS_PROTOCOL_OPERATIONS = [ + { + 'operationId': 'discoverTusCapabilities', + 'role': 'capability-discovery', + 'method': 'OPTIONS', + 'path': '/resumable/files/', + 'request': { + 'bodyKind': 'empty', + 'contentType': None, + 'headerVariants': [], + }, + 'responses': [ + { + 'statusCode': 200, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Extension', + 'name': 'tus-extension', + 'required': True, + }, + { + 'displayName': 'Tus-Max-Size', + 'name': 'tus-max-size', + 'required': True, + }, + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Tus-Version', + 'name': 'tus-version', + 'required': True, + }, + ], + }, + ], + }, + ], + }, + { + 'operationId': 'createTusUpload', + 'role': 'creation', + 'method': 'POST', + 'path': '/resumable/files/', + 'request': { + 'bodyKind': 'empty', + 'contentType': None, + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Length', + 'name': 'upload-length', + 'required': True, + }, + { + 'displayName': 'Upload-Metadata', + 'name': 'upload-metadata', + 'required': True, + }, + ], + }, + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Defer-Length', + 'name': 'upload-defer-length', + 'required': True, + }, + { + 'displayName': 'Upload-Metadata', + 'name': 'upload-metadata', + 'required': True, + }, + ], + }, + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Concat', + 'name': 'upload-concat', + 'required': True, + }, + { + 'displayName': 'Upload-Length', + 'name': 'upload-length', + 'required': True, + }, + { + 'displayName': 'Upload-Metadata', + 'name': 'upload-metadata', + 'required': False, + }, + ], + }, + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Concat', + 'name': 'upload-concat', + 'required': True, + }, + { + 'displayName': 'Upload-Metadata', + 'name': 'upload-metadata', + 'required': False, + }, + ], + }, + ], + }, + 'responses': [ + { + 'statusCode': 201, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Location', + 'name': 'location', + 'required': True, + }, + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + { + 'statusCode': 500, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + ], + }, + { + 'operationId': 'getTusUploadOffset', + 'role': 'offset-discovery', + 'method': 'HEAD', + 'path': '/resumable/files/{upload_id}', + 'request': { + 'bodyKind': 'empty', + 'contentType': None, + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + 'responses': [ + { + 'statusCode': 200, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Length', + 'name': 'upload-length', + 'required': True, + }, + { + 'displayName': 'Upload-Offset', + 'name': 'upload-offset', + 'required': True, + }, + ], + }, + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Defer-Length', + 'name': 'upload-defer-length', + 'required': True, + }, + { + 'displayName': 'Upload-Offset', + 'name': 'upload-offset', + 'required': True, + }, + ], + }, + ], + }, + ], + }, + { + 'operationId': 'patchTusUpload', + 'role': 'upload-chunk', + 'method': 'PATCH', + 'path': '/resumable/files/{upload_id}', + 'request': { + 'bodyKind': 'binary', + 'contentType': 'application/offset+octet-stream', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Content-Type', + 'name': 'content-type', + 'required': True, + }, + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Offset', + 'name': 'upload-offset', + 'required': True, + }, + ], + }, + ], + }, + 'responses': [ + { + 'statusCode': 204, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + { + 'displayName': 'Upload-Offset', + 'name': 'upload-offset', + 'required': True, + }, + ], + }, + ], + }, + { + 'statusCode': 500, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + ], + }, + { + 'operationId': 'terminateTusUpload', + 'role': 'termination', + 'method': 'DELETE', + 'path': '/resumable/files/{upload_id}', + 'request': { + 'bodyKind': 'empty', + 'contentType': None, + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + 'responses': [ + { + 'statusCode': 204, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + { + 'statusCode': 423, + 'bodyKind': 'empty', + 'headerVariants': [ + { + 'fields': [ + { + 'displayName': 'Tus-Resumable', + 'name': 'tus-resumable', + 'required': True, + }, + ], + }, + ], + }, + ], + }, + { + 'operationId': 'downloadTusUpload', + 'role': 'download', + 'method': 'GET', + 'path': '/resumable/files/{upload_id}', + 'request': { + 'bodyKind': 'empty', + 'contentType': None, + 'headerVariants': [], + }, + 'responses': [ + { + 'statusCode': 200, + 'bodyKind': 'binary', + 'headerVariants': [], + }, + ], + }, +] + +TUS_CLIENT_FEATURES = [ + { + 'conformance': { + 'scenarioIds': [ + 'singleUploadLifecycle', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Create an upload, store its URL, upload bytes, and finish successfully.', + 'featureId': 'singleUploadLifecycle', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'open-input-source', + 'summary': 'Open the caller input as a sliceable source.', + }, + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create the remote upload resource.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Upload bytes until the accepted offset reaches the known length.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'open-input-source', + 'fingerprint-input', + 'store-resume-url', + 'retry-with-backoff', + 'emit-progress', + 'abort-current-request', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'resumeFromPreviousUpload', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Resume a stored upload URL by discovering the remote offset before patching.', + 'featureId': 'resumeUpload', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'resume-from-previous-upload', + 'summary': 'Load a stored upload URL selected by fingerprint.', + }, + { + 'kind': 'operation', + 'operationId': 'getTusUploadOffset', + 'summary': 'Read the server offset for the stored upload URL.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Continue uploading from the discovered offset.', + }, + ], + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'fingerprint-input', + 'resume-from-previous-upload', + 'store-resume-url', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'deferredLengthUpload', + 'deferredLengthChunkedUpload', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Create an upload without a known length and declare the length on the final upload request.', + 'featureId': 'deferredLengthUpload', + 'flow': [ + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create the upload with deferred length.', + }, + { + 'kind': 'primitive', + 'primitive': 'defer-upload-length', + 'summary': 'Track the source until the final upload request reveals the total size.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Declare Upload-Length on the final upload request.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'defer-upload-length', + 'emit-chunk-complete', + 'emit-progress', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'creationWithUpload', + 'creationWithUploadPartialChunk', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Send the first bytes on the creation request when the server/client support it.', + 'featureId': 'creationWithUpload', + 'flow': [ + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create the upload while streaming the initial body.', + }, + { + 'kind': 'primitive', + 'primitive': 'upload-during-creation', + 'summary': 'Interpret the creation response as an accepted offset.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'upload-during-creation', + 'emit-progress', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'uploadBodyHeaders', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Send protocol-specific upload body headers whenever the client transmits file bytes.', + 'featureId': 'uploadBodyHeaders', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'send-upload-body-headers', + 'summary': 'Attach the protocol-specific upload body content type when a request has bytes.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Upload bytes with the protocol-specific body headers.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'send-upload-body-headers', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'customRequestHeaders', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Apply user-provided request headers to every upload request.', + 'featureId': 'customRequestHeaders', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'apply-custom-request-headers', + 'summary': 'Merge user-provided headers after protocol headers are prepared.', + }, + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create uploads with the configured custom headers.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Upload bytes with the configured custom headers.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'apply-custom-request-headers', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'requestIdHeaders', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Add generated request IDs after protocol and custom request headers.', + 'featureId': 'requestIdHeaders', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'add-request-id-header', + 'summary': 'Generate a request ID and apply it after custom request headers so it is authoritative.', + }, + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create uploads with a generated request ID.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Upload bytes with a generated request ID.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'add-request-id-header', + 'apply-custom-request-headers', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'overridePatchMethod', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Tunnel PATCH through POST with the method-override header.', + 'featureId': 'overridePatchMethod', + 'flow': [ + { + 'kind': 'operation', + 'operationId': 'getTusUploadOffset', + 'summary': 'Resume from the upload URL before sending bytes.', + }, + { + 'kind': 'primitive', + 'primitive': 'override-patch-method', + 'summary': 'Replace PATCH with POST while preserving the protocol operation intent.', + }, + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Upload bytes through the overridden request.', + }, + ], + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'override-patch-method', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'parallelUploadConcat', + 'parallelUploadAbortCleanup', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Split one input into partial uploads, run the parts concurrently, clean up aborted parts, and concatenate their upload URLs.', + 'featureId': 'parallelUploadConcat', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'split-parallel-upload-boundaries', + 'summary': 'Split the input into stable byte ranges.', + }, + { + 'kind': 'operation', + 'operationId': 'createTusUpload', + 'summary': 'Create partial uploads for each range.', + }, + { + 'kind': 'primitive', + 'primitive': 'concatenate-partial-uploads', + 'summary': 'Create the final upload from completed partial upload URLs.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'abort-current-request', + 'concatenate-partial-uploads', + 'emit-progress', + 'split-parallel-upload-boundaries', + 'terminate-upload', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'retryPatchAfterOffsetRecovery', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Recover from a failed chunk by reading the server offset before retrying.', + 'featureId': 'retryOffsetRecovery', + 'flow': [ + { + 'kind': 'operation', + 'operationId': 'patchTusUpload', + 'summary': 'Attempt the chunk upload.', + }, + { + 'kind': 'primitive', + 'primitive': 'recover-offset-after-error', + 'summary': 'Discover the accepted offset after a retryable failure.', + }, + { + 'kind': 'operation', + 'operationId': 'getTusUploadOffset', + 'summary': 'Use HEAD to recover the offset before retrying PATCH.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'retry-with-backoff', + 'recover-offset-after-error', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'retryPatchAfterOffsetRecovery', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Schedule retry timers and reset retry attempts after accepted progress.', + 'featureId': 'retryStateTransitions', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'schedule-retry-timer', + 'summary': 'Consume the current retry delay and restart the upload after that timer fires.', + }, + { + 'kind': 'primitive', + 'primitive': 'reset-retry-attempt-after-progress', + 'summary': 'Reset retry attempts once a later retry observes server-side offset progress.', + }, + ], + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'retry-with-backoff', + 'schedule-retry-timer', + 'reset-retry-attempt-after-progress', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'terminateWithRetry', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Terminate an upload resource and retry retryable termination failures.', + 'featureId': 'terminateUpload', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'terminate-upload', + 'summary': 'Choose server-side termination for an upload URL.', + }, + { + 'kind': 'operation', + 'operationId': 'terminateTusUpload', + 'summary': 'Delete the upload resource.', + }, + ], + 'operationIds': [ + 'terminateTusUpload', + ], + 'primitives': [ + 'terminate-upload', + 'retry-with-backoff', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'abortUpload', + 'abortUploadAfterStoredUrl', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Abort the active request, pending retry timer, and any partial uploads.', + 'featureId': 'abortUpload', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'abort-current-request', + 'summary': 'Cancel in-flight transport work without emitting user callbacks after abort.', + }, + ], + 'operationIds': [ + 'terminateTusUpload', + ], + 'primitives': [ + 'abort-current-request', + 'terminate-upload', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'singleUploadLifecycle', + 'creationWithUpload', + 'resumeFromPreviousUpload', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Expose progress and accepted-chunk callbacks from runtime upload activity.', + 'featureId': 'uploadCallbacks', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'emit-progress', + 'summary': 'Report bytes sent against known or deferred length.', + }, + { + 'kind': 'primitive', + 'primitive': 'emit-chunk-complete', + 'summary': 'Report chunk size, accepted offset, and total size after server acceptance.', + }, + { + 'kind': 'primitive', + 'primitive': 'emit-upload-url', + 'summary': 'Notify once a usable upload URL is known.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'emit-progress', + 'emit-chunk-complete', + 'emit-upload-url', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'requestLifecycleHooks', + 'retryPatchAfterOffsetRecovery', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Run before-request, after-response, and custom retry hooks around transport.', + 'featureId': 'requestLifecycleHooks', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'run-request-hooks', + 'summary': 'Call user hooks around each HTTP request/response pair.', + }, + { + 'kind': 'primitive', + 'primitive': 'customize-retry', + 'summary': 'Let user retry policy override default retry decisions.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'customize-retry', + 'run-request-hooks', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'singleUploadLifecycle', + 'resumeFromPreviousUpload', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Persist, find, resume, and optionally remove upload URLs by fingerprint.', + 'featureId': 'resumeUrlStorage', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'fingerprint-input', + 'summary': 'Derive a stable key for the input when possible.', + }, + { + 'kind': 'primitive', + 'primitive': 'store-resume-url', + 'summary': 'Persist upload URLs and partial-upload URLs for future resumption.', + }, + { + 'kind': 'primitive', + 'primitive': 'remove-stored-url-on-success', + 'summary': 'Remove stored upload URLs when configured after success or invalidation.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'fingerprint-input', + 'store-resume-url', + 'remove-stored-url-on-success', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'arrayBufferInput', + 'arrayBufferViewInput', + 'webReadableStreamInput', + 'nodeReadableStreamInput', + 'nodePathInput', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Support the reference client input/source families across runtimes.', + 'featureId': 'inputSources', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'read-browser-file', + 'summary': 'Read browser Blob/File and ArrayBuffer-family inputs.', + }, + { + 'kind': 'primitive', + 'primitive': 'read-node-stream', + 'summary': 'Read Node streams when size and chunk constraints are satisfied.', + }, + { + 'kind': 'primitive', + 'primitive': 'read-web-stream', + 'summary': 'Read Web Streams with deferred or configured size.', + }, + { + 'kind': 'primitive', + 'primitive': 'read-node-file', + 'summary': 'Read filesystem paths and fs streams, including parallel ranges.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'read-browser-file', + 'read-node-file', + 'read-node-stream', + 'read-web-stream', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'webStorageUrlStorageBackend', + 'fileUrlStorageBackend', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Support browser and file-backed URL storage implementations.', + 'featureId': 'urlStorageBackends', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'store-browser-url', + 'summary': 'Persist upload records in browser localStorage.', + }, + { + 'kind': 'primitive', + 'primitive': 'store-file-url', + 'summary': 'Persist upload records in the Node file store.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'store-browser-url', + 'store-file-url', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'ietfDraft05CreationWithUpload', + 'ietfDraft05ChunkedUploadComplete', + 'ietfDraft03ResumeWithoutKnownLength', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Select between tus v1 and supported IETF draft client protocol modes.', + 'featureId': 'protocolVersionSelection', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'select-client-protocol', + 'summary': 'Choose request headers and response expectations for the selected protocol.', + }, + ], + 'operationIds': [ + 'createTusUpload', + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'select-client-protocol', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'relativeLocationResolution', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Normalize relative Location headers against the request endpoint.', + 'featureId': 'relativeLocationResolution', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'resolve-relative-location', + 'summary': 'Resolve server Location headers with the creation endpoint as origin.', + }, + ], + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'resolve-relative-location', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'startValidationMissingInput', + 'startValidationMissingEndpointOrUploadUrl', + 'startValidationUnsupportedProtocol', + 'startValidationRetryDelaysNotArray', + 'startValidationParallelUploadsWithUploadUrl', + 'startValidationParallelUploadsWithUploadSize', + 'startValidationParallelUploadsWithDeferredLength', + 'startValidationParallelUploadsWithUploadDataDuringCreation', + 'startValidationParallelBoundariesWithoutParallelUploads', + 'startValidationParallelBoundariesLengthMismatch', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Validate option combinations before starting runtime work.', + 'featureId': 'startOptionValidation', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'validate-start-options', + 'summary': 'Reject missing inputs and incompatible parallel/deferred/resume options.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + }, + { + 'conformance': { + 'scenarioIds': [ + 'detailedCreateResponseError', + 'detailedCreateRequestError', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Attach request, response, status, body, and request ID context to errors.', + 'featureId': 'detailedErrors', + 'flow': [ + { + 'kind': 'primitive', + 'primitive': 'report-detailed-errors', + 'summary': 'Return user-facing errors with enough transport context for debugging.', + }, + ], + 'operationIds': [], + 'primitives': [ + 'report-detailed-errors', + ], + }, +] + +TUS_MANAGED_UPLOAD = { + 'capabilities': { + 'cleanup': { + 'policies': [ + 'absent-after-source-unavailable', + 'remove-owned-source-after-success', + 'remove-owned-source-after-cancel', + 'retain-owned-source-while-deferred', + 'retain-owned-source-after-permanent-failure', + 'retain-source-after-retryable-failure', + 'remove-managed-state-after-terminal-retention', + ], + }, + 'failureClassification': { + 'permanentFailures': [ + 'source-unavailable', + 'unretryable-protocol-error', + 'retry-policy-exhausted', + ], + 'retryableFailures': [ + 'retryable-protocol-error', + 'io-error', + 'network-unavailable', + ], + }, + 'networkConstraints': { + 'options': [ + 'any-network', + 'unmetered-network', + ], + }, + 'retryPolicy': { + 'controls': [ + 'max-attempts', + 'deadline', + 'progress-sensitive-budget', + 'unbounded-until-permanent-failure', + ], + 'permanentFailure': 'stop-without-retry', + 'progressReset': 'reset-budget-after-accepted-offset-advances', + }, + 'scheduling': { + 'strategies': [ + 'foreground-task', + 'process-lifetime-worker-pool', + 'durable-os-scheduler', + ], + }, + 'sourceDurability': { + 'ownedCopyCleanup': 'after-success-or-cancel', + 'strategies': [ + 'copy-to-owned-storage', + 'reference-original-source', + 'memory-only', + ], + }, + 'stateReporting': { + 'states': [ + 'pending', + 'running', + 'succeeded', + 'failed', + ], + 'terminalRetention': 'session-and-next-launch', + 'transientRetention': 'until-terminal', + }, + }, + 'conformance': { + 'scenarioIds': [ + 'managedUploadDurableRetry', + 'managedUploadPermanentFailure', + 'managedUploadRetryPolicyExhausted', + 'managedUploadSourceUnavailable', + 'managedUploadNetworkConstraint', + ], + 'status': 'covered-by-generated-scenario', + }, + 'description': 'Submit upload work that can make sources durable, schedule/resume execution, retry, report state, and clean up while reusing the raw TUS protocol features underneath.', + 'featureId': 'managedUpload', + 'flow': [ + { + 'kind': 'managed-primitive', + 'primitive': 'accept-upload-submission', + 'summary': 'Accept source, metadata, headers, endpoint, and retry/scheduling policy.', + }, + { + 'kind': 'managed-primitive', + 'primitive': 'make-source-durable', + 'summary': 'Keep the source readable according to the selected runtime durability strategy.', + }, + { + 'kind': 'managed-primitive', + 'primitive': 'schedule-upload-work', + 'summary': 'Run upload work according to the runtime scheduler capability.', + }, + { + 'featureId': 'singleUploadLifecycle', + 'kind': 'protocol-feature', + 'summary': 'Use the raw protocol upload lifecycle for each execution attempt.', + }, + { + 'featureId': 'retryOffsetRecovery', + 'kind': 'protocol-feature', + 'summary': 'Use protocol retry and offset recovery before classifying terminal failure.', + }, + { + 'kind': 'managed-primitive', + 'primitive': 'publish-upload-state', + 'summary': 'Expose pending, running, succeeded, and failed state snapshots.', + }, + { + 'kind': 'managed-primitive', + 'primitive': 'cleanup-managed-upload', + 'summary': 'Remove owned sources and terminal state according to cleanup policy.', + }, + ], + 'layer': 'feature-over-protocol', + 'primitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'apply-managed-retry-policy', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'protocolPrimitives': [ + 'store-resume-url', + 'resume-from-previous-upload', + 'recover-offset-after-error', + 'retry-with-backoff', + 'emit-progress', + 'emit-chunk-complete', + 'terminate-upload', + ], + 'runtimeProfiles': [ + { + 'networkConstraints': [ + 'any-network', + 'unmetered-network', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'sourceDurability': [ + 'copy-to-owned-storage', + 'reference-original-source', + ], + 'stateBackend': 'platform-key-value-store', + 'transportProfileId': 'java-http-url-connection', + }, + { + 'networkConstraints': [ + 'any-network', + 'unmetered-network', + ], + 'runtime': 'ios', + 'scheduler': 'durable-os-scheduler', + 'sourceDurability': [ + 'copy-to-owned-storage', + 'reference-original-source', + ], + 'stateBackend': 'platform-key-value-store', + }, + { + 'networkConstraints': [ + 'any-network', + ], + 'runtime': 'browser', + 'scheduler': 'foreground-task', + 'sourceDurability': [ + 'reference-original-source', + 'memory-only', + ], + 'stateBackend': 'web-storage', + }, + { + 'networkConstraints': [ + 'any-network', + ], + 'runtime': 'java', + 'scheduler': 'process-lifetime-worker-pool', + 'sourceDurability': [ + 'copy-to-owned-storage', + 'reference-original-source', + ], + 'stateBackend': 'filesystem', + 'transportProfileId': 'java-http-url-connection', + }, + { + 'networkConstraints': [ + 'any-network', + ], + 'runtime': 'node', + 'scheduler': 'process-lifetime-worker-pool', + 'sourceDurability': [ + 'copy-to-owned-storage', + 'reference-original-source', + 'memory-only', + ], + 'stateBackend': 'filesystem', + }, + { + 'networkConstraints': [ + 'any-network', + ], + 'runtime': 'react-native', + 'scheduler': 'foreground-task', + 'sourceDurability': [ + 'reference-original-source', + 'memory-only', + ], + 'stateBackend': 'platform-key-value-store', + }, + ], + 'scenarios': [ + { + 'proofs': [ + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'afterAcceptedOffset': 7, + 'kind': 'io-error', + 'phase': 'after-accepted-offset', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': { + 'Location': 'https://tus.io/uploads/managed-durable-retry', + }, + 'statusCode': 201, + }, + 'url': 'endpoint', + }, + { + 'bodySize': 7, + 'headers': { + 'Upload-Offset': '0', + }, + 'operationId': 'patchTusUpload', + 'response': { + 'headers': { + 'Upload-Offset': '7', + }, + 'statusCode': 204, + }, + 'url': 'upload', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 1, + 'requests': [ + { + 'headers': {}, + 'operationId': 'getTusUploadOffset', + 'response': { + 'headers': { + 'Upload-Length': '14', + 'Upload-Offset': '7', + }, + 'statusCode': 200, + }, + 'url': 'upload', + }, + { + 'bodySize': 7, + 'headers': { + 'Upload-Offset': '7', + }, + 'operationId': 'patchTusUpload', + 'response': { + 'headers': { + 'Upload-Offset': '14', + }, + 'statusCode': 204, + }, + 'url': 'upload', + }, + ], + 'stateAfterAttempt': 'succeeded', + }, + ], + 'cleanup': { + 'ownedSource': 'remove-owned-source-after-success', + 'resumeUrl': 'remove-after-success', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello managed!', + 'fingerprint': 'managed-durable-retry-fingerprint', + 'metadata': { + 'filename': 'managed.txt', + }, + 'uploadPath': 'managed-durable-retry', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'kind': 'terminal', + 'state': 'succeeded', + }, + 'retryDelays': [ + 0, + ], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + 'running', + 'succeeded', + ], + 'runtime': 'java', + 'scheduler': 'process-lifetime-worker-pool', + 'stateBackend': 'filesystem', + }, + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'afterAcceptedOffset': 7, + 'kind': 'io-error', + 'phase': 'after-accepted-offset', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': { + 'Location': 'https://tus.io/uploads/managed-durable-retry', + }, + 'statusCode': 201, + }, + 'url': 'endpoint', + }, + { + 'bodySize': 7, + 'headers': { + 'Upload-Offset': '0', + }, + 'operationId': 'patchTusUpload', + 'response': { + 'headers': { + 'Upload-Offset': '7', + }, + 'statusCode': 204, + }, + 'url': 'upload', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 1, + 'requests': [ + { + 'headers': {}, + 'operationId': 'getTusUploadOffset', + 'response': { + 'headers': { + 'Upload-Length': '14', + 'Upload-Offset': '7', + }, + 'statusCode': 200, + }, + 'url': 'upload', + }, + { + 'bodySize': 7, + 'headers': { + 'Upload-Offset': '7', + }, + 'operationId': 'patchTusUpload', + 'response': { + 'headers': { + 'Upload-Offset': '14', + }, + 'statusCode': 204, + }, + 'url': 'upload', + }, + ], + 'stateAfterAttempt': 'succeeded', + }, + ], + 'cleanup': { + 'ownedSource': 'remove-owned-source-after-success', + 'resumeUrl': 'remove-after-success', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello managed!', + 'fingerprint': 'managed-durable-retry-fingerprint', + 'metadata': { + 'filename': 'managed.txt', + }, + 'uploadPath': 'managed-durable-retry', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'kind': 'terminal', + 'state': 'succeeded', + }, + 'retryDelays': [ + 0, + ], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + 'running', + 'succeeded', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'stateBackend': 'platform-key-value-store', + }, + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'apply-managed-retry-policy', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'scenarioId': 'managedUploadDurableRetry', + 'summary': 'Submit a durable source, survive scheduler/process interruption, resume by stored upload URL, and finish with cleanup.', + }, + { + 'proofs': [ + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'unretryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 400, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'retain-owned-source-after-permanent-failure', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello failure!', + 'fingerprint': 'managed-permanent-failure-fingerprint', + 'metadata': { + 'filename': 'managed-permanent-failure.txt', + }, + 'uploadPath': 'managed-permanent-failure', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'unretryable-protocol-error', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + ], + 'runtime': 'java', + 'scheduler': 'process-lifetime-worker-pool', + 'stateBackend': 'filesystem', + }, + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'unretryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 400, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'retain-owned-source-after-permanent-failure', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello failure!', + 'fingerprint': 'managed-permanent-failure-fingerprint', + 'metadata': { + 'filename': 'managed-permanent-failure.txt', + }, + 'uploadPath': 'managed-permanent-failure', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'unretryable-protocol-error', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'stateBackend': 'platform-key-value-store', + }, + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'scenarioId': 'managedUploadPermanentFailure', + 'summary': 'Classify unretryable protocol failures as terminal without further retry.', + }, + { + 'proofs': [ + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 1, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 2, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'retain-owned-source-after-permanent-failure', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello retries!', + 'fingerprint': 'managed-retry-exhausted-fingerprint', + 'metadata': { + 'filename': 'managed-retry-exhausted.txt', + }, + 'uploadPath': 'managed-retry-exhausted', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'retry-policy-exhausted', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [ + 0, + 0, + ], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + 'running', + 'failed', + 'running', + 'failed', + ], + 'runtime': 'java', + 'scheduler': 'process-lifetime-worker-pool', + 'stateBackend': 'filesystem', + }, + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 1, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + { + 'attemptIndex': 2, + 'failure': { + 'kind': 'retryable-protocol-error', + 'phase': 'during-protocol-request', + }, + 'requests': [ + { + 'bodySize': 0, + 'headers': { + 'Upload-Length': '14', + }, + 'operationId': 'createTusUpload', + 'response': { + 'headers': {}, + 'statusCode': 500, + }, + 'url': 'endpoint', + }, + ], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'retain-owned-source-after-permanent-failure', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello retries!', + 'fingerprint': 'managed-retry-exhausted-fingerprint', + 'metadata': { + 'filename': 'managed-retry-exhausted.txt', + }, + 'uploadPath': 'managed-retry-exhausted', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'retry-policy-exhausted', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [ + 0, + 0, + ], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + 'running', + 'failed', + 'running', + 'failed', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'stateBackend': 'platform-key-value-store', + }, + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'apply-managed-retry-policy', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'scenarioId': 'managedUploadRetryPolicyExhausted', + 'summary': 'Retry transient protocol failures up to the managed retry budget and then classify the upload as terminally failed.', + }, + { + 'proofs': [ + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'source-unavailable', + 'phase': 'before-protocol-request', + }, + 'requests': [], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'absent-after-source-unavailable', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello missing!', + 'fingerprint': 'managed-source-unavailable-fingerprint', + 'metadata': { + 'filename': 'managed-source-unavailable.txt', + }, + 'uploadPath': 'managed-source-unavailable', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'source-unavailable', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [], + 'sourceAvailability': 'missing-before-durable-copy', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + ], + 'runtime': 'java', + 'scheduler': 'process-lifetime-worker-pool', + 'stateBackend': 'filesystem', + }, + { + 'attempts': [ + { + 'attemptIndex': 0, + 'failure': { + 'kind': 'source-unavailable', + 'phase': 'before-protocol-request', + }, + 'requests': [], + 'stateAfterAttempt': 'failed', + }, + ], + 'cleanup': { + 'ownedSource': 'absent-after-source-unavailable', + 'resumeUrl': 'absent-after-permanent-failure', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello missing!', + 'fingerprint': 'managed-source-unavailable-fingerprint', + 'metadata': { + 'filename': 'managed-source-unavailable.txt', + }, + 'uploadPath': 'managed-source-unavailable', + }, + 'network': { + 'current': 'unmetered-network', + 'decision': 'start-upload-work', + 'required': 'any-network', + }, + 'outcome': { + 'failure': 'source-unavailable', + 'kind': 'terminal', + 'state': 'failed', + }, + 'retryDelays': [], + 'sourceAvailability': 'missing-before-durable-copy', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + 'running', + 'failed', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'stateBackend': 'platform-key-value-store', + }, + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'scenarioId': 'managedUploadSourceUnavailable', + 'summary': 'Classify source disappearance before protocol requests as terminal without issuing a TUS request.', + }, + { + 'proofs': [ + { + 'attempts': [], + 'cleanup': { + 'ownedSource': 'retain-owned-source-while-deferred', + 'resumeUrl': 'absent-while-deferred', + }, + 'input': { + 'chunkSize': 7, + 'content': 'hello later!', + 'fingerprint': 'managed-network-constraint-fingerprint', + 'metadata': { + 'filename': 'managed-network-constraint.txt', + }, + 'uploadPath': 'managed-network-constraint', + }, + 'network': { + 'current': 'metered-network', + 'decision': 'defer-until-network-constraint-satisfied', + 'required': 'unmetered-network', + }, + 'outcome': { + 'kind': 'deferred', + 'reason': 'network-constraint-unsatisfied', + 'state': 'pending', + }, + 'retryDelays': [], + 'sourceAvailability': 'available', + 'sourceDurability': 'copy-to-owned-storage', + 'states': [ + 'pending', + ], + 'runtime': 'android', + 'scheduler': 'durable-os-scheduler', + 'stateBackend': 'platform-key-value-store', + }, + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'publish-upload-state', + ], + 'scenarioId': 'managedUploadNetworkConstraint', + 'summary': 'Honor network constraints before starting or resuming upload work.', + }, + ], +} + +TUS_MANAGED_UPLOAD_PROOF_CASES = [ + { + 'featureId': 'managedUpload', + 'layer': 'feature-over-protocol', + 'proofRuntimes': [ + 'java', + 'android', + ], + 'protocolFeatureIds': [ + 'singleUploadLifecycle', + 'retryOffsetRecovery', + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'apply-managed-retry-policy', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'runtimeProfiles': [ + 'android', + 'ios', + 'browser', + 'java', + 'node', + 'react-native', + ], + 'scenarioId': 'managedUploadDurableRetry', + }, + { + 'featureId': 'managedUpload', + 'layer': 'feature-over-protocol', + 'proofRuntimes': [ + 'java', + 'android', + ], + 'protocolFeatureIds': [ + 'singleUploadLifecycle', + 'retryOffsetRecovery', + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'runtimeProfiles': [ + 'android', + 'ios', + 'browser', + 'java', + 'node', + 'react-native', + ], + 'scenarioId': 'managedUploadPermanentFailure', + }, + { + 'featureId': 'managedUpload', + 'layer': 'feature-over-protocol', + 'proofRuntimes': [ + 'java', + 'android', + ], + 'protocolFeatureIds': [ + 'singleUploadLifecycle', + 'retryOffsetRecovery', + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'run-protocol-upload', + 'apply-managed-retry-policy', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'runtimeProfiles': [ + 'android', + 'ios', + 'browser', + 'java', + 'node', + 'react-native', + ], + 'scenarioId': 'managedUploadRetryPolicyExhausted', + }, + { + 'featureId': 'managedUpload', + 'layer': 'feature-over-protocol', + 'proofRuntimes': [ + 'java', + 'android', + ], + 'protocolFeatureIds': [ + 'singleUploadLifecycle', + 'retryOffsetRecovery', + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'classify-failure', + 'publish-upload-state', + 'cleanup-managed-upload', + ], + 'runtimeProfiles': [ + 'android', + 'ios', + 'browser', + 'java', + 'node', + 'react-native', + ], + 'scenarioId': 'managedUploadSourceUnavailable', + }, + { + 'featureId': 'managedUpload', + 'layer': 'feature-over-protocol', + 'proofRuntimes': [ + 'android', + ], + 'protocolFeatureIds': [ + 'singleUploadLifecycle', + 'retryOffsetRecovery', + ], + 'requiredPrimitives': [ + 'accept-upload-submission', + 'make-source-durable', + 'schedule-upload-work', + 'publish-upload-state', + ], + 'runtimeProfiles': [ + 'android', + 'ios', + 'browser', + 'java', + 'node', + 'react-native', + ], + 'scenarioId': 'managedUploadNetworkConstraint', + }, +] + +TUS_CLIENT_CONFORMANCE_SCENARIOS = [ + { + 'behavior': 'single-upload-lifecycle', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/generated-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'fingerprint:contract-single-fingerprint', + 'upload-url-available', + 'url-storage-add:contract-single-fingerprint:https://tus.io/uploads/generated-contract', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'fingerprint', + 'upload-url-available', + 'url-storage-add', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'singleUploadLifecycle', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'open-input-source', + 'fingerprint-input', + 'store-resume-url', + 'retry-with-backoff', + 'emit-progress', + 'abort-current-request', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/generated-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/generated-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-single-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': True, + 'storedUpload': None, + }, + }, + 'scenarioId': 'singleUploadLifecycle', + }, + { + 'behavior': 'creation-with-upload', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/creation-with-upload-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'progress', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'creationWithUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadDataDuringCreation', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'upload-during-creation', + 'emit-progress', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/creation-with-upload-contract', + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'creationWithUpload', + }, + { + 'behavior': 'creation-with-upload-partial-chunk', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/creation-with-upload-partial-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:5:11', + 'upload-url-available', + 'chunk-complete:5:5:11', + 'progress:5:11', + 'progress:10:11', + 'chunk-complete:5:10:11', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'progress', + 'upload-url-available', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'creationWithUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 5, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadDataDuringCreation', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'upload-during-creation', + 'emit-progress', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/creation-with-upload-partial-contract', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': 'https://tus.io/uploads/creation-with-upload-partial-contract', + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/creation-with-upload-partial-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 1, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-final-chunk', + 'uploadUrl': 'https://tus.io/uploads/creation-with-upload-partial-contract', + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/creation-with-upload-partial-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'creationWithUploadPartialChunk', + }, + { + 'behavior': 'creation-with-upload', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/ietf-draft-05-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'progress', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'protocolVersionSelection', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'protocol', + 'value': 'ietf-draft-05', + }, + { + 'key': 'uploadDataDuringCreation', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'select-client-protocol', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Length': '11', + 'Upload-Complete': '?1', + 'Content-Type': 'application/partial-upload', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Location': 'https://tus.io/uploads/ietf-draft-05-contract', + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'ietfDraft05CreationWithUpload', + }, + { + 'behavior': 'upload-body-headers', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:5:11', + 'chunk-complete:5:5:11', + 'progress:5:11', + 'progress:10:11', + 'chunk-complete:5:10:11', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'upload-url-available', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'protocolVersionSelection', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 5, + }, + { + 'key': 'protocol', + 'value': 'ietf-draft-05', + }, + { + 'key': 'uploadUrl', + 'value': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'select-client-protocol', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + }, + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Complete': '?0', + 'Content-Type': 'application/partial-upload', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + }, + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Complete': '?0', + 'Content-Type': 'application/partial-upload', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + }, + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': 1, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Complete': '?1', + 'Content-Type': 'application/partial-upload', + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-final-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-05-chunked-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'ietfDraft05ChunkedUploadComplete', + }, + { + 'behavior': 'upload-body-headers', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/ietf-draft-03-resume-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'upload-url-available', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'protocolVersionSelection', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 6, + }, + { + 'key': 'protocol', + 'value': 'ietf-draft-03', + }, + { + 'key': 'uploadUrl', + 'value': 'https://tus.io/uploads/ietf-draft-03-resume-contract', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'select-client-protocol', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-03-resume-contract', + }, + { + 'absentHeaders': [ + 'Content-Type', + 'Tus-Resumable', + ], + 'abort': False, + 'bodySize': 6, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Complete': '?1', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': 'exact', + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/ietf-draft-03-resume-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'ietfDraft03ResumeWithoutKnownLength', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: no file or stream to upload provided', + 'completionReason': 'missingInput', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + ], + 'inputSource': { + 'content': '', + 'kind': 'none', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationMissingInput', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: neither an endpoint or an upload URL is provided', + 'completionReason': 'missingEndpointOrUploadUrl', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationMissingEndpointOrUploadUrl', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: unsupported protocol tus-v9', + 'completionReason': 'unsupportedProtocol', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'protocol', + 'value': 'tus-v9', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationUnsupportedProtocol', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: the `retryDelays` option must either be an array or null', + 'completionReason': 'retryDelaysNotArray', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'rawOptions', + 'value': { + 'retryDelays': 44, + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationRetryDelaysNotArray', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: cannot use the `uploadUrl` option when parallelUploads is enabled', + 'completionReason': 'parallelUploadsWithUploadUrl', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + { + 'key': 'uploadUrl', + 'value': 'https://tus.io/uploads/start-validation-upload-url', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelUploadsWithUploadUrl', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: cannot use the `uploadSize` option when parallelUploads is enabled', + 'completionReason': 'parallelUploadsWithUploadSize', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + { + 'key': 'uploadSize', + 'value': 11, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelUploadsWithUploadSize', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: cannot use the `uploadLengthDeferred` option when parallelUploads is enabled', + 'completionReason': 'parallelUploadsWithDeferredLength', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + { + 'key': 'uploadLengthDeferred', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelUploadsWithDeferredLength', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: cannot use the `uploadDataDuringCreation` option when parallelUploads is enabled', + 'completionReason': 'parallelUploadsWithUploadDataDuringCreation', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + { + 'key': 'uploadDataDuringCreation', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelUploadsWithUploadDataDuringCreation', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: cannot use the `parallelUploadBoundaries` option when `parallelUploads` is disabled', + 'completionReason': 'parallelBoundariesWithoutParallelUploads', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploadBoundaries', + 'value': [ + { + 'end': 5, + 'start': 0, + }, + ], + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelBoundariesWithoutParallelUploads', + }, + { + 'behavior': 'start-option-validation', + 'completionKind': 'error', + 'completionMessage': 'tus: the `parallelUploadBoundaries` must have the same length as the value of `parallelUploads`', + 'completionReason': 'parallelBoundariesLengthMismatch', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'startOptionValidation', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + { + 'key': 'parallelUploadBoundaries', + 'value': [ + { + 'end': 5, + 'start': 0, + }, + ], + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [], + 'primitives': [ + 'validate-start-options', + ], + 'requests': [], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'startValidationParallelBoundariesLengthMismatch', + }, + { + 'behavior': 'detailed-error', + 'completionKind': 'error', + 'completionMessage': 'tus: unexpected response while creating upload, originated from request (method: POST, url: https://tus.io/uploads, response code: 500, response text: server_error, request id: contract-request-id)', + 'completionReason': 'unexpectedCreateResponse', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'detailedErrors', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Request-ID': 'contract-request-id', + }, + }, + { + 'key': 'rawOptions', + 'value': { + 'retryDelays': None, + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'report-detailed-errors', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + 'X-Request-ID': 'contract-request-id', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': 'server_error', + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 500, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'detailedCreateResponseError', + }, + { + 'behavior': 'detailed-error', + 'completionKind': 'error', + 'completionMessage': 'tus: failed to create upload, caused by Error: socket down, originated from request (method: POST, url: https://tus.io/uploads, response code: n/a, response text: n/a, request id: contract-request-id)', + 'completionReason': 'createUploadRequestFailed', + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'detailedErrors', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Request-ID': 'contract-request-id', + }, + }, + { + 'key': 'rawOptions', + 'value': { + 'retryDelays': None, + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'report-detailed-errors', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': 'socket down', + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + 'X-Request-ID': 'contract-request-id', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': None, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'detailedCreateRequestError', + }, + { + 'behavior': 'upload-body-headers', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/upload-body-headers-contract', + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'uploadBodyHeaders', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'send-upload-body-headers', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/upload-body-headers-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/upload-body-headers-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'uploadBodyHeaders', + }, + { + 'behavior': 'custom-request-headers', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/custom-headers-contract', + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'customRequestHeaders', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Tus-Contract': 'custom-header', + 'X-Tus-Trace': 'trace-123', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'apply-custom-request-headers', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + 'X-Tus-Contract': 'custom-header', + 'X-Tus-Trace': 'trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/custom-headers-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + 'X-Tus-Contract': 'custom-header', + 'X-Tus-Trace': 'trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/custom-headers-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-custom-headers-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'customRequestHeaders', + }, + { + 'behavior': 'request-id-headers', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/request-id-contract', + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'requestIdHeaders', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Request-ID': 'custom-request-id', + }, + }, + { + 'key': 'addRequestId', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'add-request-id-header', + 'apply-custom-request-headers', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + 'X-Request-ID': '00000000-0000-4000-8000-000000000000', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/request-id-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + 'X-Request-ID': '00000000-0000-4000-8000-000000000000', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/request-id-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': True, + 'generatedRequestId': '00000000-0000-4000-8000-000000000000', + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'requestIdHeaders', + }, + { + 'behavior': 'resume-from-previous-upload', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/resume-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'fingerprint:contract-resume-fingerprint', + 'url-storage-find:contract-resume-fingerprint:1', + 'fingerprint:contract-resume-fingerprint', + 'upload-url-available', + 'progress:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + 'url-storage-remove:tus::contract-resume-fingerprint::1337', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'fingerprint', + 'url-storage-find', + 'upload-url-available', + 'progress', + 'chunk-complete', + 'url-storage-remove', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'expectedPreviousUploadCount': 1, + 'kind': 'resume-from-previous-upload', + 'selectedPreviousUploadIndex': 0, + }, + ], + 'phase': 'beforeStart', + }, + ], + 'featureId': 'resumeUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'removeFingerprintOnSuccess', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'fingerprint-input', + 'resume-from-previous-upload', + 'store-resume-url', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': 'recover-upload-offset', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads/resume-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 6, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/resume-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-resume-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': True, + 'storedUpload': { + 'fingerprint': 'contract-resume-fingerprint', + 'uploadUrl': 'https://tus.io/uploads/resume-contract', + 'urlStorageKey': 'tus::contract-resume-fingerprint::1337', + }, + }, + }, + 'scenarioId': 'resumeFromPreviousUpload', + }, + { + 'behavior': 'relative-location-resolution', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/files/relative-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'upload-url-available', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'relativeLocationResolution', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/files/', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'resolve-relative-location', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'relative-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/files/', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/files/relative-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'relativeLocationResolution', + }, + { + 'behavior': 'array-buffer-input', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/array-buffer-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:array-buffer:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'source-open', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'inputSources', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'array-buffer', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-browser-file', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/array-buffer-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/array-buffer-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'arrayBufferInput', + }, + { + 'behavior': 'array-buffer-view-input', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/array-buffer-view-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:array-buffer-view:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'source-open', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'inputSources', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'array-buffer-view', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-browser-file', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/array-buffer-view-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/array-buffer-view-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'arrayBufferViewInput', + }, + { + 'behavior': 'web-readable-stream-input', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/web-stream-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:web-readable-stream:null', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'source-open', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'inputSources', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 100, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadLengthDeferred', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'web-readable-stream', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-web-stream', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Upload-Length', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/web-stream-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/web-stream-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'webReadableStreamInput', + }, + { + 'behavior': 'node-readable-stream-input', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/node-stream-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:node-readable-stream:null', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'source-open', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'inputSources', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 100, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadLengthDeferred', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'node-readable-stream', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-node-stream', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Upload-Length', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/node-stream-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': None, + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/node-stream-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'nodeReadableStreamInput', + 'runtimes': [ + 'node', + ], + }, + { + 'behavior': 'node-path-input', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/node-path-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:node-path-reference:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'source-open', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'inputSources', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'node-path-reference', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-node-file', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/node-path-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/node-path-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'nodePathInput', + 'runtimes': [ + 'node', + ], + }, + { + 'behavior': 'deferred-length-upload', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/deferred-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'upload-url-available', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'deferredLengthUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 100, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadLengthDeferred', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'web-readable-stream', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'defer-upload-length', + 'emit-progress', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Upload-Length', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/deferred-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/deferred-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'deferredLengthUpload', + }, + { + 'behavior': 'deferred-length-upload', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/deferred-chunked-contract', + 'eventKeyAlternativeGroups': [ + [], + [ + 'progress:0:11', + ], + [ + 'progress:5:11', + ], + [ + 'chunk-complete:5:5:11', + ], + [ + 'progress:5:11', + ], + [ + 'progress:10:11', + ], + [ + 'chunk-complete:5:10:11', + ], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:null', + 'progress:5:null', + 'chunk-complete:5:5:null', + 'progress:5:null', + 'progress:10:null', + 'chunk-complete:5:10:null', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'upload-url-available', + 'progress', + 'chunk-complete', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [], + 'featureId': 'deferredLengthUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 5, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'uploadLengthDeferred', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'defer-upload-length', + 'emit-chunk-complete', + 'emit-progress', + ], + 'requests': [ + { + 'absentHeaders': [ + 'Upload-Length', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/deferred-chunked-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': 0, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/deferred-chunked-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/deferred-chunked-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 1, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '10', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-final-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/deferred-chunked-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'deferredLengthChunkedUpload', + }, + { + 'behavior': 'override-patch-method', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/override-contract', + 'eventKeyAlternativeGroups': [], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [], + 'eventKinds': [], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'overridePatchMethod', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'overridePatchMethod', + 'value': True, + }, + { + 'key': 'uploadUrl', + 'value': 'https://tus.io/uploads/override-contract', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'override-patch-method', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '3', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': 'recover-upload-offset', + 'uploadUrl': 'https://tus.io/uploads/override-contract', + 'url': 'upload', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads/override-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 8, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '3', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': 'https://tus.io/uploads/override-contract', + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/override-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-override-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'overridePatchMethod', + }, + { + 'behavior': 'parallel-upload-concat', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/parallel-final', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:5:11', + 'chunk-complete:5:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + ], + 'eventKinds': [ + 'progress', + 'chunk-complete', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'gateId': 'parallel-patches', + 'heldRequestIndexes': [ + 2, + 3, + ], + 'kind': 'release-after-all-started', + 'releaseAfterRequestIndexes': [ + 2, + 3, + ], + 'timeoutMs': 2000, + }, + ], + 'phase': 'serverRequestGates', + }, + ], + 'featureId': 'parallelUploadConcat', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'foo': 'hello', + }, + }, + { + 'key': 'metadataForPartialUploads', + 'value': { + 'test': 'world', + }, + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'createTusUpload', + 'patchTusUpload', + 'patchTusUpload', + 'createTusUpload', + ], + 'primitives': [ + 'concatenate-partial-uploads', + 'emit-progress', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Metadata': 'test d29ybGQ=', + 'Upload-Concat': 'partial', + 'Upload-Length': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/parallel-part-1', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-partial-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Metadata': 'test d29ybGQ=', + 'Upload-Concat': 'partial', + 'Upload-Length': '6', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/parallel-part-2', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-partial-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-partial-chunk', + 'uploadUrl': 'https://tus.io/uploads/parallel-part-1', + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/parallel-part-1', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 6, + 'bodyStart': 5, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '6', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-partial-chunk', + 'uploadUrl': 'https://tus.io/uploads/parallel-part-2', + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/parallel-part-2', + }, + { + 'absentHeaders': [ + 'Upload-Length', + ], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Metadata': 'foo aGVsbG8=', + 'Upload-Concat': 'final;https://tus.io/uploads/parallel-part-1 https://tus.io/uploads/parallel-part-2', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/parallel-final', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-final-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 4, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'parallelUploadConcat', + }, + { + 'behavior': 'parallel-upload-abort-cleanup', + 'completionKind': 'aborted', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:3', + ], + 'eventKinds': [ + 'request-abort', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'gateId': 'parallel-cleanup-patches', + 'heldRequestIndexes': [ + 2, + 3, + ], + 'kind': 'release-after-all-started', + 'releaseAfterRequestIndexes': [ + 2, + 3, + ], + 'timeoutMs': 2000, + }, + ], + 'phase': 'serverRequestGates', + }, + ], + 'featureId': 'parallelUploadConcat', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadataForPartialUploads', + 'value': { + 'test': 'world', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + }, + { + 'key': 'overridePatchMethod', + 'value': True, + }, + { + 'key': 'parallelUploads', + 'value': 2, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'createTusUpload', + 'patchTusUpload', + 'patchTusUpload', + 'terminateTusUpload', + 'terminateTusUpload', + ], + 'primitives': [ + 'abort-current-request', + 'terminate-upload', + 'concatenate-partial-uploads', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Metadata': 'test d29ybGQ=', + 'Upload-Concat': 'partial', + 'Upload-Length': '5', + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/parallel-cleanup-part-1', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-partial-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Metadata': 'test d29ybGQ=', + 'Upload-Concat': 'partial', + 'Upload-Length': '6', + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/parallel-cleanup-part-2', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-partial-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': 0, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 500, + }, + 'role': 'upload-partial-chunk', + 'uploadUrl': 'https://tus.io/uploads/parallel-cleanup-part-1', + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/parallel-cleanup-part-1', + }, + { + 'absentHeaders': [], + 'abort': True, + 'bodySize': 6, + 'bodyStart': 5, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': None, + 'role': 'upload-partial-chunk', + 'uploadUrl': 'https://tus.io/uploads/parallel-cleanup-part-2', + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/parallel-cleanup-part-2', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'terminateTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 204, + }, + 'role': 'terminate-upload', + 'uploadUrl': 'https://tus.io/uploads/parallel-cleanup-part-1', + 'url': 'upload', + 'requestIndex': 4, + 'expectedUrl': 'https://tus.io/uploads/parallel-cleanup-part-1', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'X-Tus-Contract': 'parallel-cleanup-policy', + 'X-Tus-Trace': 'parallel-cleanup-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'terminateTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 204, + }, + 'role': 'terminate-upload', + 'uploadUrl': 'https://tus.io/uploads/parallel-cleanup-part-2', + 'url': 'upload', + 'requestIndex': 5, + 'expectedUrl': 'https://tus.io/uploads/parallel-cleanup-part-2', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': True, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-parallel-cleanup-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'parallelUploadAbortCleanup', + }, + { + 'behavior': 'retry-patch-after-offset-recovery', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/retry-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'should-retry:0:true', + 'retry-schedule:0', + 'should-retry:0:true', + 'retry-schedule:0', + ], + 'eventKinds': [ + 'should-retry', + 'retry-schedule', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'retryOffsetRecovery', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'retryDelays', + 'value': [ + 0, + ], + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + 'getTusUploadOffset', + 'patchTusUpload', + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'retry-with-backoff', + 'recover-offset-after-error', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/retry-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 500, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/retry-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': 'recover-upload-offset', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/retry-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 6, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 500, + }, + 'role': 'retry-upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/retry-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': 'recover-upload-offset', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 4, + 'expectedUrl': 'https://tus.io/uploads/retry-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 6, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-final-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 5, + 'expectedUrl': 'https://tus.io/uploads/retry-contract', + }, + ], + 'retryDecisions': [ + { + 'decision': True, + 'retryAttempt': 0, + }, + { + 'decision': True, + 'retryAttempt': 0, + }, + ], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'retryPatchAfterOffsetRecovery', + }, + { + 'behavior': 'request-lifecycle-hooks', + 'completionKind': 'success', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/request-hooks-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'before-request:0', + 'after-response:0', + 'success', + 'source-close', + ], + 'eventKinds': [ + 'before-request', + 'after-response', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [], + 'featureId': 'requestLifecycleHooks', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'uploadUrl', + 'value': 'https://tus.io/uploads/request-hooks-contract', + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'getTusUploadOffset', + ], + 'primitives': [ + 'run-request-hooks', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'getTusUploadOffset', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Offset': '11', + }, + 'headersSpecified': True, + 'statusCode': 200, + }, + 'role': 'recover-upload-offset', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads/request-hooks-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'requestLifecycleHooks', + }, + { + 'behavior': 'abort-upload', + 'completionKind': 'aborted', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': None, + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:0', + ], + 'eventKinds': [ + 'request-abort', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'kind': 'cancel-upload', + 'requestIndex': 0, + }, + ], + 'phase': 'onRequestStart', + }, + ], + 'featureId': 'abortUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + ], + 'primitives': [ + 'abort-current-request', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': True, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': None, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'abortUpload', + }, + { + 'behavior': 'abort-upload-after-stored-url', + 'completionKind': 'aborted', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/abort-terminate-contract', + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:1', + ], + 'eventKinds': [ + 'request-abort', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'kind': 'cancel-upload', + 'requestIndex': 1, + }, + ], + 'phase': 'onRequestStart', + }, + ], + 'featureId': 'abortUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'headers', + 'value': { + 'X-Tus-Contract': 'abort-policy', + 'X-Tus-Trace': 'abort-trace-123', + }, + }, + { + 'key': 'overridePatchMethod', + 'value': True, + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + 'terminateTusUpload', + ], + 'primitives': [ + 'abort-current-request', + 'terminate-upload', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + 'X-Tus-Contract': 'abort-policy', + 'X-Tus-Trace': 'abort-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/abort-terminate-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': True, + 'bodySize': 11, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + 'X-Tus-Contract': 'abort-policy', + 'X-Tus-Trace': 'abort-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': None, + 'role': 'abort-upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/abort-terminate-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'X-Tus-Contract': 'abort-policy', + 'X-Tus-Trace': 'abort-trace-123', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'terminateTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 204, + }, + 'role': 'terminate-upload', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/abort-terminate-contract', + }, + ], + 'retryDecisions': [], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': True, + }, + 'fingerprint': { + 'install': True, + 'value': 'contract-abort-terminate-fingerprint', + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'abortUploadAfterStoredUrl', + }, + { + 'behavior': 'terminate-with-retry', + 'completionKind': 'terminated', + 'completionMessage': None, + 'completionReason': None, + 'completionUploadUrl': 'https://tus.io/uploads/terminate-contract', + 'eventKeyAlternativeGroups': [ + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'should-retry:0:true', + 'retry-schedule:0', + ], + 'eventKinds': [ + 'should-retry', + 'retry-schedule', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'executionActionPhases': [ + { + 'actions': [ + { + 'kind': 'abort-upload', + 'terminateUpload': True, + }, + ], + 'phase': 'onChunkComplete', + }, + ], + 'featureId': 'terminateUpload', + 'inputOptionEntries': [ + { + 'key': 'endpointUrl', + 'value': 'https://tus.io/uploads', + }, + { + 'key': 'chunkSize', + 'value': 5, + }, + { + 'key': 'metadata', + 'value': { + 'filename': 'hello.txt', + }, + }, + { + 'key': 'retryDelays', + 'value': [ + 0, + 0, + ], + }, + ], + 'inputSource': { + 'content': 'hello world', + 'kind': 'blob', + }, + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + 'terminateTusUpload', + 'terminateTusUpload', + ], + 'primitives': [ + 'terminate-upload', + 'retry-with-backoff', + ], + 'requests': [ + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'createTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Location': 'https://tus.io/uploads/terminate-contract', + }, + 'headersSpecified': True, + 'statusCode': 201, + }, + 'role': 'create-upload', + 'uploadUrl': None, + 'url': 'endpoint', + 'requestIndex': 0, + 'expectedUrl': 'https://tus.io/uploads', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': 5, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'headersSpecified': True, + 'method': None, + 'operationId': 'patchTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': { + 'Upload-Offset': '5', + }, + 'headersSpecified': True, + 'statusCode': 204, + }, + 'role': 'upload-chunk', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 1, + 'expectedUrl': 'https://tus.io/uploads/terminate-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'terminateTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 423, + }, + 'role': 'terminate-upload', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 2, + 'expectedUrl': 'https://tus.io/uploads/terminate-contract', + }, + { + 'absentHeaders': [], + 'abort': False, + 'bodySize': None, + 'bodyStart': None, + 'errorMessage': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'method': None, + 'operationId': 'terminateTusUpload', + 'response': { + 'body': None, + 'headerMode': None, + 'headers': {}, + 'headersSpecified': False, + 'statusCode': 204, + }, + 'role': 'retry-terminate-upload', + 'uploadUrl': None, + 'url': 'upload', + 'requestIndex': 3, + 'expectedUrl': 'https://tus.io/uploads/terminate-contract', + }, + ], + 'retryDecisions': [ + { + 'decision': True, + 'retryAttempt': 0, + }, + ], + 'runtimeSetup': { + 'abort': { + 'terminateUpload': False, + }, + 'fingerprint': { + 'install': False, + 'value': None, + }, + 'requestId': { + 'enabled': False, + 'generatedRequestId': None, + }, + 'urlStorage': { + 'install': False, + 'storedUpload': None, + }, + }, + 'scenarioId': 'terminateWithRetry', + }, +] diff --git a/tests/storage_file b/tests/storage_file deleted file mode 100644 index dfe2bda..0000000 --- a/tests/storage_file +++ /dev/null @@ -1 +0,0 @@ -{"_default": {"1": {"key": "size:1082--md5:cbb679e9dbcf82224fe3bc5fdc881f06", "url": "http://tusd.tusdemo.net/files/foo_bar"}, "2": {"key": "size:49588--md5:ae275d47f1ef9aed4902b0251455e627", "url": "http://tusd.tusdemo.net/files/foo_bar"}}} \ No newline at end of file diff --git a/tests/test_client.py b/tests/test_client.py index b55b862..f0abf46 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,11 +1,50 @@ import unittest +from io import BytesIO import responses from tusclient import client +from tusclient.exceptions import TusCommunicationError +from tusclient.fingerprint.interface import Fingerprint +from tusclient.protocol_generated import ( + DEFAULT_REQUEST_HEADERS, + CREATE_UPLOAD_METHOD, + LOCATION_HEADER_NAME, + TERMINATE_UPLOAD_METHOD, + TUS_PROTOCOL_REQUEST_HEADERS, + UPLOAD_BODY_CONTENT_TYPE, + UPLOAD_BODY_CONTENT_TYPE_HEADER_NAME, + UPLOAD_LENGTH_HEADER_NAME, + UPLOAD_OFFSET_HEADER_NAME, + upload_body_headers, +) +from tusclient.request_lifecycle import RequestLifecycleHooks +from tusclient.storage.interface import Storage from tusclient.uploader import Uploader, AsyncUploader +class MemoryStorage(Storage): + def __init__(self): + self.values = {} + + def get_item(self, key): + return self.values.get(key) + + def set_item(self, key, value): + self.values[key] = value + + def remove_item(self, key): + self.values.pop(key, None) + + +class FixedFingerprint(Fingerprint): + def __init__(self, value): + self.value = value + + def get_fingerprint(self, fs): + return self.value + + class TusClientTest(unittest.TestCase): def setUp(self): self.client = client.TusClient('http://tusd.tusdemo.net/files/', @@ -14,6 +53,7 @@ def setUp(self): def test_instance_attributes(self): self.assertEqual(self.client.url, 'http://tusd.tusdemo.net/files/') self.assertEqual(self.client.headers, {'foo': 'bar'}) + self.assertFalse(self.client.add_request_id) def test_set_headers(self): self.client.set_headers({'foo': 'bar tender'}) @@ -23,6 +63,197 @@ def test_set_headers(self): self.client.set_headers({'food': 'at the bar'}) self.assertEqual(self.client.headers, {'foo': 'bar tender', 'food': 'at the bar'}) + def test_request_id_header_toggle(self): + self.client.enable_request_id_header() + self.assertTrue(self.client.add_request_id) + + self.client.disable_request_id_header() + self.assertFalse(self.client.add_request_id) + + @responses.activate + def test_terminate_upload(self): + url = 'http://tusd.tusdemo.net/files/15acd89eabdf5738ffc' + events = [] + + def before_request(context): + events.append(('before', context.method, context.url)) + context.headers['x-hook'] = 'before' + + def after_response(context, response): + events.append(('after', context.method, response.status_code)) + + self.client.set_request_hooks( + RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ) + ) + responses.add(TERMINATE_UPLOAD_METHOD, url, status=204) + + response = self.client.terminate_upload(url) + request = responses.calls[0].request + + self.assertEqual(response.status_code, 204) + self.assertEqual(request.method, TERMINATE_UPLOAD_METHOD) + for header_name, header_value in DEFAULT_REQUEST_HEADERS.items(): + self.assertEqual(request.headers[header_name], header_value) + self.assertEqual(request.headers['x-hook'], 'before') + self.assertEqual( + events, + [ + ('before', TERMINATE_UPLOAD_METHOD, url), + ('after', TERMINATE_UPLOAD_METHOD, 204), + ], + ) + + @responses.activate + def test_terminate_upload_non_success_status(self): + url = 'http://tusd.tusdemo.net/files/15acd89eabdf5738ffc' + responses.add(TERMINATE_UPLOAD_METHOD, url, status=404, body='gone') + + with self.assertRaises(TusCommunicationError) as context: + self.client.terminate_upload(url) + + self.assertEqual(context.exception.status_code, 404) + self.assertEqual(context.exception.response_content, b'gone') + + @responses.activate + def test_abort_upload_terminates_and_removes_stored_url(self): + upload_url = 'http://tusd.tusdemo.net/files/abort' + storage = MemoryStorage() + uploader = self.client.uploader( + file_stream=BytesIO(b'hello'), + fingerprinter=FixedFingerprint('abort-fingerprint'), + store_url=True, + url_storage=storage, + ) + uploader.set_url(upload_url) + responses.add(TERMINATE_UPLOAD_METHOD, upload_url, status=204) + + response = self.client.abort_upload(uploader, terminate_upload=True) + + self.assertEqual(response.status_code, 204) + self.assertTrue(uploader.is_aborted()) + self.assertIsNone(storage.get_item('abort-fingerprint')) + + @responses.activate + def test_create_upload_with_data(self): + upload_url = 'http://tusd.tusdemo.net/files/creation-with-upload' + events = [] + + def before_request(context): + events.append(('before', context.method, context.url)) + context.headers['x-hook'] = 'before' + + def after_response(context, response): + events.append(('after', context.method, response.status_code)) + + def validate_create_request(request): + self.assertEqual(request.body, b'hello') + self.assertEqual(request.headers[UPLOAD_LENGTH_HEADER_NAME], '5') + self.assertEqual( + request.headers[UPLOAD_BODY_CONTENT_TYPE_HEADER_NAME], + UPLOAD_BODY_CONTENT_TYPE, + ) + self.assertEqual(request.headers['x-hook'], 'before') + for header_name, header_value in DEFAULT_REQUEST_HEADERS.items(): + self.assertEqual(request.headers[header_name], header_value) + + return ( + 201, + { + LOCATION_HEADER_NAME: upload_url, + UPLOAD_OFFSET_HEADER_NAME: '5', + }, + '', + ) + + self.client.set_request_hooks( + RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ) + ) + responses.add_callback( + CREATE_UPLOAD_METHOD, + self.client.url, + callback=validate_create_request, + ) + + uploader = self.client.create_upload_with_data( + 5, + file_stream=BytesIO(b'hello'), + chunk_size=5, + metadata={}, + ) + + self.assertIsInstance(uploader, Uploader) + self.assertEqual(uploader.url, upload_url) + self.assertEqual(uploader.offset, 5) + self.assertEqual( + events, + [ + ('before', CREATE_UPLOAD_METHOD, self.client.url), + ('after', CREATE_UPLOAD_METHOD, 201), + ], + ) + + @responses.activate + def test_create_upload_with_data_uses_selected_protocol_headers(self): + upload_url = 'http://tusd.tusdemo.net/files/ietf-draft-05' + protocol = 'ietf-draft-05' + protocol_request_headers = TUS_PROTOCOL_REQUEST_HEADERS[protocol] + body_headers = upload_body_headers(protocol, done=True) + + def validate_create_request(request): + self.assertEqual(request.body, b'hello') + self.assertNotIn('Tus-Resumable', request.headers) + for header_name, header_value in protocol_request_headers.items(): + self.assertEqual(request.headers[header_name], header_value) + for header_name, header_value in body_headers.items(): + self.assertEqual(request.headers[header_name], header_value) + + return ( + 201, + { + LOCATION_HEADER_NAME: upload_url, + UPLOAD_OFFSET_HEADER_NAME: '5', + }, + '', + ) + + responses.add_callback( + CREATE_UPLOAD_METHOD, + self.client.url, + callback=validate_create_request, + ) + + uploader = self.client.create_upload_with_data( + 5, + file_stream=BytesIO(b'hello'), + chunk_size=5, + metadata={}, + protocol=protocol, + ) + + self.assertEqual(uploader.url, upload_url) + self.assertEqual(uploader.offset, 5) + + @responses.activate + def test_create_upload_with_data_non_success_status(self): + responses.add(CREATE_UPLOAD_METHOD, self.client.url, status=400, body='bad') + + with self.assertRaises(TusCommunicationError) as context: + self.client.create_upload_with_data( + 5, + file_stream=BytesIO(b'hello'), + chunk_size=5, + metadata={}, + ) + + self.assertEqual(context.exception.status_code, 400) + self.assertEqual(context.exception.response_content, b'bad') + @responses.activate def test_uploader(self): url = 'http://tusd.tusdemo.net/files/15acd89eabdf5738ffc' diff --git a/tests/test_detailed_error.py b/tests/test_detailed_error.py new file mode 100644 index 0000000..0239dcf --- /dev/null +++ b/tests/test_detailed_error.py @@ -0,0 +1,77 @@ +from io import BytesIO + +import pytest +import requests +import responses + +from tusclient import client +from tusclient.exceptions import TusDetailedError +from tusclient.protocol_generated import CREATE_UPLOAD_METHOD, REQUEST_ID_HEADER_NAME + + +def uploader_for(tus_client): + return tus_client.uploader( + file_stream=BytesIO(b'hello world'), + metadata={'filename': 'hello.txt'}, + ) + + +@responses.activate +def test_create_upload_response_error_preserves_context(): + endpoint = 'https://tus.io/uploads' + tus_client = client.TusClient( + endpoint, + headers={REQUEST_ID_HEADER_NAME: 'contract-request-id'}, + ) + responses.add(CREATE_UPLOAD_METHOD, endpoint, status=500, body='server_error') + + with pytest.raises(TusDetailedError) as error: + uploader_for(tus_client).create_url() + + assert str(error.value) == ( + 'tus: unexpected response while creating upload, originated from request ' + '(method: POST, url: https://tus.io/uploads, response code: 500, ' + 'response text: server_error, request id: contract-request-id)' + ) + assert error.value.causing_error is None + assert error.value.original_request_method == CREATE_UPLOAD_METHOD + assert error.value.original_request_url == endpoint + assert error.value.original_request_id == 'contract-request-id' + assert error.value.original_response_body == 'server_error' + assert error.value.original_response_present is True + assert error.value.original_response_status == 500 + assert error.value.status_code == 500 + assert error.value.response_content == b'server_error' + + +def test_create_upload_request_error_preserves_context(monkeypatch): + endpoint = 'https://tus.io/uploads' + tus_client = client.TusClient( + endpoint, + headers={REQUEST_ID_HEADER_NAME: 'contract-request-id'}, + ) + + def failing_post(url, **kwargs): + assert url == endpoint + assert kwargs['headers'][REQUEST_ID_HEADER_NAME] == 'contract-request-id' + raise requests.exceptions.ConnectionError('socket down') + + monkeypatch.setattr(requests, 'post', failing_post) + + with pytest.raises(TusDetailedError) as error: + uploader_for(tus_client).create_url() + + assert str(error.value) == ( + 'tus: failed to create upload, caused by Error: socket down, ' + 'originated from request (method: POST, url: https://tus.io/uploads, ' + 'response code: n/a, response text: n/a, request id: contract-request-id)' + ) + assert str(error.value.causing_error) == 'socket down' + assert error.value.original_request_method == CREATE_UPLOAD_METHOD + assert error.value.original_request_url == endpoint + assert error.value.original_request_id == 'contract-request-id' + assert error.value.original_response_body is None + assert error.value.original_response_present is False + assert error.value.original_response_status is None + assert error.value.status_code is None + assert error.value.response_content is None diff --git a/tests/test_examples.py b/tests/test_examples.py new file mode 100644 index 0000000..e3eb7dc --- /dev/null +++ b/tests/test_examples.py @@ -0,0 +1,10 @@ +"""Smoke checks for checked-in examples.""" + +import py_compile +from pathlib import Path + + +def test_api2_devdock_examples_compile(): + examples_root = Path(__file__).parent.parent / "examples" + for example in examples_root.glob("api2-devdock-*/main.py"): + py_compile.compile(str(example), doraise=True) diff --git a/tests/test_filestorage.py b/tests/test_filestorage.py index b77ffed..94c91a8 100644 --- a/tests/test_filestorage.py +++ b/tests/test_filestorage.py @@ -1,6 +1,7 @@ import unittest import os +from tusclient.protocol_generated import url_storage_fingerprint_prefix from tusclient.storage import filestorage @@ -24,6 +25,12 @@ def test_set_get_remove_item(self): self.assertEqual(self.storage.get_item(key), url) self.assertEqual(self.storage.get_item(key_2), url_2) + self.assertEqual(self.storage.count(), 2) + self.assertTrue(self.storage.keys()[0].startswith(url_storage_fingerprint_prefix(key))) + self.assertTrue(self.storage.keys()[1].startswith(url_storage_fingerprint_prefix(key_2))) + + self.storage.set_item(key, url) + self.assertEqual(self.storage.count(), 2) self.storage.remove_item(key) self.assertIsNone(self.storage.get_item(key)) diff --git a/tests/test_generated_conformance_events.py b/tests/test_generated_conformance_events.py new file mode 100644 index 0000000..55def33 --- /dev/null +++ b/tests/test_generated_conformance_events.py @@ -0,0 +1,721 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +import unittest + +from tests.generated_protocol_contract import ( + TUS_CLIENT_CONFORMANCE_SCENARIOS, + TUS_CLIENT_FEATURES, + TUS_MANAGED_UPLOAD, + TUS_MANAGED_UPLOAD_PROOF_CASES, +) + + +CASES = [ + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'fingerprint:contract-single-fingerprint', + 'upload-url-available', + 'url-storage-add:contract-single-fingerprint:https://tus.io/uploads/generated-contract', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'singleUploadLifecycle', + 'scenarioId': 'singleUploadLifecycle', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'creationWithUpload', + 'scenarioId': 'creationWithUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:5:11', + 'upload-url-available', + 'chunk-complete:5:5:11', + 'progress:5:11', + 'progress:10:11', + 'chunk-complete:5:10:11', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'creationWithUpload', + 'scenarioId': 'creationWithUploadPartialChunk', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'upload-url-available', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'protocolVersionSelection', + 'scenarioId': 'ietfDraft05CreationWithUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:5:11', + 'chunk-complete:5:5:11', + 'progress:5:11', + 'progress:10:11', + 'chunk-complete:5:10:11', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'protocolVersionSelection', + 'scenarioId': 'ietfDraft05ChunkedUploadComplete', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'protocolVersionSelection', + 'scenarioId': 'ietfDraft03ResumeWithoutKnownLength', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'fingerprint:contract-resume-fingerprint', + 'url-storage-find:contract-resume-fingerprint:1', + 'fingerprint:contract-resume-fingerprint', + 'upload-url-available', + 'progress:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + 'url-storage-remove:tus::contract-resume-fingerprint::1337', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'resumeUpload', + 'scenarioId': 'resumeFromPreviousUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'relativeLocationResolution', + 'scenarioId': 'relativeLocationResolution', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:array-buffer:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'inputSources', + 'scenarioId': 'arrayBufferInput', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:array-buffer-view:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'inputSources', + 'scenarioId': 'arrayBufferViewInput', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:web-readable-stream:null', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'inputSources', + 'scenarioId': 'webReadableStreamInput', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:node-readable-stream:null', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'inputSources', + 'scenarioId': 'nodeReadableStreamInput', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'source-open:node-path-reference:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'inputSources', + 'scenarioId': 'nodePathInput', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'deferredLengthUpload', + 'scenarioId': 'deferredLengthUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [ + 'progress:0:11', + ], + [ + 'progress:5:11', + ], + [ + 'chunk-complete:5:5:11', + ], + [ + 'progress:5:11', + ], + [ + 'progress:10:11', + ], + [ + 'chunk-complete:5:10:11', + ], + [], + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'upload-url-available', + 'progress:0:null', + 'progress:5:null', + 'chunk-complete:5:5:null', + 'progress:5:null', + 'progress:10:null', + 'chunk-complete:5:10:null', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'deferredLengthUpload', + 'scenarioId': 'deferredLengthChunkedUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'eventKeys': [ + 'progress:5:11', + 'chunk-complete:5:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'featureId': 'parallelUploadConcat', + 'scenarioId': 'parallelUploadConcat', + }, + { + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:3', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'parallelUploadConcat', + 'scenarioId': 'parallelUploadAbortCleanup', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'should-retry:0:true', + 'retry-schedule:0', + 'should-retry:0:true', + 'retry-schedule:0', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'retryOffsetRecovery', + 'scenarioId': 'retryPatchAfterOffsetRecovery', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'before-request:0', + 'after-response:0', + 'success', + 'source-close', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'requestLifecycleHooks', + 'scenarioId': 'requestLifecycleHooks', + }, + { + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:0', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'abortUpload', + 'scenarioId': 'abortUpload', + }, + { + 'eventKeyAlternativeGroups': [ + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'request-abort:1', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'abortUpload', + 'scenarioId': 'abortUploadAfterStoredUrl', + }, + { + 'eventKeyAlternativeGroups': [ + [], + [], + ], + 'eventKeyExtraPrefixes': [], + 'eventKeys': [ + 'should-retry:0:true', + 'retry-schedule:0', + ], + 'eventPolicy': { + 'matching': 'exact', + }, + 'featureId': 'terminateUpload', + 'scenarioId': 'terminateWithRetry', + }, +] + +PROOF_CASES = [ + { + 'behavior': 'single-upload-lifecycle', + 'completionKind': 'success', + 'featureId': 'singleUploadLifecycle', + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'open-input-source', + 'fingerprint-input', + 'store-resume-url', + 'retry-with-backoff', + 'emit-progress', + 'abort-current-request', + ], + 'profile': 'urlStorageCreateFlow', + 'scenarioId': 'singleUploadLifecycle', + }, + { + 'behavior': 'custom-request-headers', + 'completionKind': 'success', + 'featureId': 'customRequestHeaders', + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'apply-custom-request-headers', + ], + 'profile': 'customRequestHeaders', + 'scenarioId': 'customRequestHeaders', + }, + { + 'behavior': 'override-patch-method', + 'completionKind': 'success', + 'featureId': 'overridePatchMethod', + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'override-patch-method', + ], + 'profile': 'overridePatchMethod', + 'scenarioId': 'overridePatchMethod', + }, + { + 'behavior': 'node-path-input', + 'completionKind': 'success', + 'featureId': 'inputSources', + 'operationIds': [ + 'createTusUpload', + 'patchTusUpload', + ], + 'primitives': [ + 'read-node-file', + ], + 'profile': 'nodePathFileUpload', + 'scenarioId': 'nodePathInput', + }, + { + 'behavior': 'resume-from-previous-upload', + 'completionKind': 'success', + 'featureId': 'resumeUpload', + 'operationIds': [ + 'getTusUploadOffset', + 'patchTusUpload', + ], + 'primitives': [ + 'fingerprint-input', + 'resume-from-previous-upload', + 'store-resume-url', + ], + 'profile': 'resumeFromPreviousUpload', + 'scenarioId': 'resumeFromPreviousUpload', + }, +] + + +def client_feature(feature_id): + for feature in TUS_CLIENT_FEATURES: + if feature["featureId"] == feature_id: + return feature + raise AssertionError("Missing generated TUS client feature: {}".format(feature_id)) + + +def client_scenario(scenario_id): + for scenario in TUS_CLIENT_CONFORMANCE_SCENARIOS: + if scenario["scenarioId"] == scenario_id: + return scenario + raise AssertionError("Missing generated TUS client scenario: {}".format(scenario_id)) + + +def managed_upload_scenario(scenario_id): + for scenario in TUS_MANAGED_UPLOAD["scenarios"]: + if scenario["scenarioId"] == scenario_id: + return scenario + raise AssertionError("Missing generated TUS managed-upload scenario: {}".format(scenario_id)) + + +class GeneratedTusConformanceEventsTest(unittest.TestCase): + def test_generated_scenario_event_keys(self): + for case in CASES: + scenario = client_scenario(case["scenarioId"]) + feature = client_feature(case["featureId"]) + + self.assertEqual(scenario["featureId"], case["featureId"]) + self.assertIn(scenario["scenarioId"], feature["conformance"]["scenarioIds"]) + self.assertEqual( + scenario["eventKeys"], + case["eventKeys"], + ) + self.assertEqual( + scenario["eventKeyAlternativeGroups"], + case["eventKeyAlternativeGroups"], + ) + self.assertEqual( + scenario["eventKeyExtraPrefixes"], + case["eventKeyExtraPrefixes"], + ) + self.assertEqual( + scenario["eventPolicy"], + case["eventPolicy"], + ) + + def test_generated_proof_profile_scenarios(self): + for case in PROOF_CASES: + scenario = client_scenario(case["scenarioId"]) + feature = client_feature(case["featureId"]) + + self.assertEqual(scenario["behavior"], case["behavior"]) + self.assertEqual(scenario["completionKind"], case["completionKind"]) + self.assertEqual(scenario["featureId"], case["featureId"]) + self.assertIn(scenario["scenarioId"], feature["conformance"]["scenarioIds"]) + self.assertEqual(scenario["operationIds"], case["operationIds"]) + self.assertEqual(scenario["primitives"], case["primitives"]) + + def test_generated_managed_upload_proof_scenarios(self): + for case in TUS_MANAGED_UPLOAD_PROOF_CASES: + scenario = managed_upload_scenario(case["scenarioId"]) + + self.assertEqual(TUS_MANAGED_UPLOAD["featureId"], case["featureId"]) + self.assertEqual(TUS_MANAGED_UPLOAD["layer"], case["layer"]) + self.assertEqual(scenario["requiredPrimitives"], case["requiredPrimitives"]) + for primitive in case["requiredPrimitives"]: + self.assertIn(primitive, TUS_MANAGED_UPLOAD["primitives"]) + for feature_id in case["protocolFeatureIds"]: + client_feature(feature_id) + self.assertEqual( + [profile["runtime"] for profile in TUS_MANAGED_UPLOAD["runtimeProfiles"]], + case["runtimeProfiles"], + ) diff --git a/tests/test_generated_protocol_contract.py b/tests/test_generated_protocol_contract.py new file mode 100644 index 0000000..a4e28d7 --- /dev/null +++ b/tests/test_generated_protocol_contract.py @@ -0,0 +1,160 @@ +import io +import unittest +from urllib.parse import urljoin + +import responses + +from tests.generated_protocol_contract import ( + TUS_CLIENT_FEATURES, + TUS_PROTOCOL_OPERATIONS, + TUS_WIRE_VERSIONS, +) +from tusclient.client import TusClient +from tusclient.protocol_generated import ( + DEFAULT_PROTOCOL_VERSION, + DEFAULT_REQUEST_HEADERS, + DEFAULT_RESPONSE_HEADERS, +) +from tusclient.uploader.baseuploader import BaseUploader + + +def default_wire_version(): + versions = [version for version in TUS_WIRE_VERSIONS if version["default"]] + if len(versions) != 1: + raise AssertionError("Generated TUS protocol contract must have one default wire version") + return versions[0]["value"] + + +def protocol_operation(operation_id): + for operation in TUS_PROTOCOL_OPERATIONS: + if operation["operationId"] == operation_id: + return operation + raise AssertionError("Missing generated TUS protocol operation: {}".format(operation_id)) + + +def client_feature(feature_id): + for feature in TUS_CLIENT_FEATURES: + if feature["featureId"] == feature_id: + return feature + raise AssertionError("Missing generated TUS client feature: {}".format(feature_id)) + + +def response_for(operation, status_code): + for response in operation["responses"]: + if response["statusCode"] == status_code: + return response + raise AssertionError( + "Missing generated response status {} for {}".format( + status_code, + operation["operationId"], + ) + ) + + +def response_headers_for(response, overrides): + headers = {} + variant = response["headerVariants"][0] + for field in variant["fields"]: + if not field["required"]: + continue + if field["displayName"] in overrides: + headers[field["displayName"]] = overrides[field["displayName"]] + continue + headers[field["displayName"]] = DEFAULT_RESPONSE_HEADERS[field["displayName"]] + return headers + + +def request_header(request, field): + return request.headers.get(field["displayName"]) or request.headers.get(field["name"]) + + +class GeneratedProtocolContractTest(unittest.TestCase): + def test_runtime_default_headers_are_generated_contract_headers(self): + self.assertEqual(BaseUploader.DEFAULT_HEADERS, DEFAULT_REQUEST_HEADERS) + + @responses.activate + def test_drives_create_and_patch_lifecycle_assertions_from_generated_contract(self): + self.assertEqual(DEFAULT_PROTOCOL_VERSION, default_wire_version()) + + lifecycle = client_feature("singleUploadLifecycle") + create_operation = protocol_operation(lifecycle["operationIds"][0]) + patch_operation = protocol_operation(lifecycle["operationIds"][2]) + client = TusClient("http://tusd.tusdemo.net/files/") + upload_url = urljoin(client.url, "generated-contract") + + create_response = response_for(create_operation, 201) + responses.add( + create_operation["method"], + client.url, + adding_headers=response_headers_for( + create_response, + {"Location": upload_url}, + ), + status=create_response["statusCode"], + ) + + patch_response = response_for(patch_operation, 204) + responses.add( + patch_operation["method"], + upload_url, + adding_headers=response_headers_for( + patch_response, + {"Upload-Offset": "5"}, + ), + status=patch_response["statusCode"], + ) + + uploader = client.uploader( + file_stream=io.BytesIO(b"hello"), + chunk_size=5, + metadata={"filename": "hello.txt"}, + ) + uploader.upload() + + create_request = responses.calls[0].request + self.assertEqual(create_request.method, create_operation["method"]) + for field in create_operation["request"]["headerVariants"][0]["fields"]: + self.assertIsNotNone(request_header(create_request, field)) + self.assertEqual( + request_header( + create_request, + {"displayName": "Tus-Resumable", "name": "tus-resumable"}, + ), + default_wire_version(), + ) + self.assertEqual( + request_header( + create_request, + {"displayName": "Upload-Length", "name": "upload-length"}, + ), + "5", + ) + self.assertEqual( + request_header( + create_request, + {"displayName": "Upload-Metadata", "name": "upload-metadata"}, + ), + "filename aGVsbG8udHh0", + ) + + patch_request = responses.calls[1].request + self.assertEqual(patch_request.method, patch_operation["method"]) + for field in patch_operation["request"]["headerVariants"][0]["fields"]: + self.assertIsNotNone(request_header(patch_request, field)) + self.assertEqual( + request_header( + patch_request, + {"displayName": "Content-Type", "name": "content-type"}, + ), + patch_operation["request"]["contentType"], + ) + self.assertEqual( + request_header( + patch_request, + {"displayName": "Upload-Offset", "name": "upload-offset"}, + ), + "0", + ) + self.assertEqual(patch_request.body, b"hello") + self.assertEqual(uploader.url, upload_url) + self.assertEqual(uploader.offset, 5) diff --git a/tests/test_generated_runtime_events.py b/tests/test_generated_runtime_events.py new file mode 100644 index 0000000..a1a4f05 --- /dev/null +++ b/tests/test_generated_runtime_events.py @@ -0,0 +1,724 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +import io +import unittest + +import responses + +from tusclient.client import TusClient +from tusclient.fingerprint.interface import Fingerprint +from tusclient.protocol_generated import DEFAULT_REQUEST_HEADERS, DEFAULT_RESPONSE_HEADERS +from tusclient.storage.interface import Storage + + +CASES = [ + { + 'chunkSize': 11, + 'content': 'hello world', + 'endpointHasTrailingSlash': False, + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'endpointUrl': 'https://tus.io/uploads', + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'locationHeaderName': 'Location', + 'locationHeaderKind': 'absolute', + 'metadata': { + 'filename': 'hello.txt', + }, + 'removeFingerprintOnSuccess': False, + 'requests': [ + { + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'method': 'POST', + 'responseHeaders': { + 'Location': 'https://tus.io/uploads/generated-contract', + }, + 'statusCode': 201, + 'url': 'endpoint', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '11', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + ], + 'scenarioId': 'singleUploadLifecycle', + 'storedUpload': None, + 'uploadLengthDeferred': False, + 'uploadPath': 'generated-contract', + 'uploadUrl': 'https://tus.io/uploads/generated-contract', + }, + { + 'beforeStartActions': [ + { + 'expectedPreviousUploadCount': 1, + 'kind': 'resume-from-previous-upload', + 'selectedPreviousUploadIndex': 0, + }, + ], + 'chunkSize': 6, + 'content': 'hello world', + 'endpointHasTrailingSlash': False, + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'endpointUrl': 'https://tus.io/uploads', + 'eventKeys': [ + 'progress:5:11', + 'progress:11:11', + 'chunk-complete:6:11:11', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'locationHeaderName': 'Location', + 'locationHeaderKind': 'stored', + 'metadata': {}, + 'removeFingerprintOnSuccess': True, + 'requests': [ + { + 'headers': {}, + 'method': 'HEAD', + 'responseHeaders': { + 'Upload-Length': '11', + 'Upload-Offset': '5', + }, + 'statusCode': 200, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '11', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + ], + 'scenarioId': 'resumeFromPreviousUpload', + 'storedUpload': { + 'fingerprint': 'contract-resume-fingerprint', + 'uploadUrl': 'https://tus.io/uploads/resume-contract', + 'urlStorageKey': 'tus::contract-resume-fingerprint::1337', + }, + 'uploadLengthDeferred': False, + 'uploadPath': 'resume-contract', + 'uploadUrl': 'https://tus.io/uploads/resume-contract', + }, + { + 'chunkSize': 11, + 'content': 'hello world', + 'endpointHasTrailingSlash': True, + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'endpointUrl': 'https://tus.io/files/', + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + ], + 'eventPolicy': { + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'locationHeaderName': 'Location', + 'locationHeaderKind': 'relative', + 'metadata': { + 'filename': 'hello.txt', + }, + 'removeFingerprintOnSuccess': False, + 'requests': [ + { + 'headers': { + 'Upload-Length': '11', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'method': 'POST', + 'responseHeaders': { + 'Location': 'relative-contract', + }, + 'statusCode': 201, + 'url': 'endpoint', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '11', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + ], + 'scenarioId': 'relativeLocationResolution', + 'storedUpload': None, + 'uploadLengthDeferred': False, + 'uploadPath': 'relative-contract', + 'uploadUrl': 'https://tus.io/files/relative-contract', + }, + { + 'chunkSize': 100, + 'content': 'hello world', + 'endpointHasTrailingSlash': False, + 'eventKeyAlternativeGroups': [ + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'endpointUrl': 'https://tus.io/uploads', + 'eventKeys': [ + 'progress:0:11', + 'progress:11:11', + 'chunk-complete:11:11:11', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'locationHeaderName': 'Location', + 'locationHeaderKind': 'absolute', + 'metadata': { + 'filename': 'hello.txt', + }, + 'removeFingerprintOnSuccess': False, + 'requests': [ + { + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'method': 'POST', + 'responseHeaders': { + 'Location': 'https://tus.io/uploads/deferred-contract', + }, + 'statusCode': 201, + 'url': 'endpoint', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '11', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + ], + 'scenarioId': 'deferredLengthUpload', + 'storedUpload': None, + 'uploadLengthDeferred': True, + 'uploadPath': 'deferred-contract', + 'uploadUrl': 'https://tus.io/uploads/deferred-contract', + }, + { + 'chunkSize': 5, + 'content': 'hello world', + 'endpointHasTrailingSlash': False, + 'eventKeyAlternativeGroups': [ + [ + 'progress:0:11', + ], + [ + 'progress:5:11', + ], + [ + 'chunk-complete:5:5:11', + ], + [ + 'progress:5:11', + ], + [ + 'progress:10:11', + ], + [ + 'chunk-complete:5:10:11', + ], + [], + [], + [], + ], + 'eventKeyExtraPrefixes': [ + 'progress:', + ], + 'endpointUrl': 'https://tus.io/uploads', + 'eventKeys': [ + 'progress:0:null', + 'progress:5:null', + 'chunk-complete:5:5:null', + 'progress:5:null', + 'progress:10:null', + 'chunk-complete:5:10:null', + 'progress:10:11', + 'progress:11:11', + 'chunk-complete:1:11:11', + ], + 'eventPolicy': { + 'deferredLengthBytesTotal': 'allow-known-total-before-declaration', + 'matching': 'exact-except-allowed-extra-events', + 'progress': 'milestone', + 'transportProgress': 'may-emit-extra-samples', + }, + 'locationHeaderName': 'Location', + 'locationHeaderKind': 'absolute', + 'metadata': { + 'filename': 'hello.txt', + }, + 'removeFingerprintOnSuccess': False, + 'requests': [ + { + 'headers': { + 'Upload-Defer-Length': '1', + 'Upload-Metadata': 'filename aGVsbG8udHh0', + }, + 'method': 'POST', + 'responseHeaders': { + 'Location': 'https://tus.io/uploads/deferred-chunked-contract', + }, + 'statusCode': 201, + 'url': 'endpoint', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '0', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '5', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '5', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '10', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + { + 'headers': { + 'Upload-Length': '11', + 'Content-Type': 'application/offset+octet-stream', + 'Upload-Offset': '10', + }, + 'method': 'PATCH', + 'responseHeaders': { + 'Upload-Offset': '11', + }, + 'statusCode': 204, + 'url': 'upload', + 'includesDefaultProtocolRequestHeaders': True, + 'includesDefaultProtocolResponseHeaders': True, + }, + ], + 'scenarioId': 'deferredLengthChunkedUpload', + 'storedUpload': None, + 'uploadLengthDeferred': True, + 'uploadPath': 'deferred-chunked-contract', + 'uploadUrl': 'https://tus.io/uploads/deferred-chunked-contract', + }, +] + + +class GeneratedTusStorage(Storage): + def __init__(self, values): + self.values = dict(values) + + def get_item(self, key): + return self.values.get(key) + + def set_item(self, key, value): + self.values[key] = value + + def remove_item(self, key): + self.values.pop(key, None) + + +class GeneratedTusFingerprinter(Fingerprint): + def __init__(self, fingerprint): + self.fingerprint = fingerprint + + def get_fingerprint(self, fs): + return self.fingerprint + + +def format_event_value(value): + return 'null' if value is None else str(value) + + +GENERATED_TUS_EVENT_KEY_PART_SEPARATOR = ':' +GENERATED_TUS_SUCCESS_REMOVE_STORED_URL_BEFORE_HOOK = True +GENERATED_TUS_SUCCESS_REMOVE_STORED_URL_REQUIRES_OPTION = True +GENERATED_TUS_URL_STORAGE_REMOVE_ON_SUCCESS_ENABLED = True +GENERATED_TUS_URL_STORAGE_REMOVE_ON_SUCCESS_REQUIRES_OPTION = True + + +def generated_tus_event_key(*parts): + return GENERATED_TUS_EVENT_KEY_PART_SEPARATOR.join(parts) + + +def generated_tus_event_key_after_response(request_index): + return generated_tus_event_key('after-response', request_index) + +def generated_tus_event_key_before_request(request_index): + return generated_tus_event_key('before-request', request_index) + +def generated_tus_event_key_chunk_complete(chunk_size, bytes_accepted, bytes_total): + return generated_tus_event_key('chunk-complete', chunk_size, bytes_accepted, bytes_total) + +def generated_tus_event_key_fingerprint(fingerprint): + return generated_tus_event_key('fingerprint', fingerprint) + +def generated_tus_event_key_progress(bytes_sent, bytes_total): + return generated_tus_event_key('progress', bytes_sent, bytes_total) + +def generated_tus_event_key_request_abort(request_index): + return generated_tus_event_key('request-abort', request_index) + +def generated_tus_event_key_retry_schedule(delay): + return generated_tus_event_key('retry-schedule', delay) + +def generated_tus_event_key_should_retry(retry_attempt, decision): + return generated_tus_event_key('should-retry', retry_attempt, decision) + +def generated_tus_event_key_source_close(): + return generated_tus_event_key('source-close') + +def generated_tus_event_key_source_open(input_kind, size): + return generated_tus_event_key('source-open', input_kind, size) + +def generated_tus_event_key_success(): + return generated_tus_event_key('success') + +def generated_tus_event_key_upload_url_available(): + return generated_tus_event_key('upload-url-available') + +def generated_tus_event_key_url_storage_add(fingerprint, upload_url): + return generated_tus_event_key('url-storage-add', fingerprint, upload_url) + +def generated_tus_event_key_url_storage_find(fingerprint, count): + return generated_tus_event_key('url-storage-find', fingerprint, count) + +def generated_tus_event_key_url_storage_remove(url_storage_key): + return generated_tus_event_key('url-storage-remove', url_storage_key) + + +def record_progress(events): + def on_progress(bytes_sent, bytes_total): + events.append( + generated_tus_event_key_progress( + format_event_value(bytes_sent), + format_event_value(bytes_total), + ) + ) + return on_progress + + +def record_chunk_complete(events): + def on_chunk_complete(chunk_size, bytes_accepted, bytes_total): + events.append( + generated_tus_event_key_chunk_complete( + format_event_value(chunk_size), + format_event_value(bytes_accepted), + format_event_value(bytes_total), + ) + ) + return on_chunk_complete + + +def has_allowed_extra_event_prefix(event_key, prefixes): + for prefix in prefixes: + if event_key.startswith(prefix): + return True + + return False + + +def resume_before_start_action(case): + action = None + for candidate in case.get('beforeStartActions', []): + if candidate['kind'] != 'resume-from-previous-upload': + raise AssertionError( + '{} uses unsupported generated beforeStart action {}'.format( + case['scenarioId'], candidate['kind'] + ) + ) + + if action is not None: + raise AssertionError( + '{} defines more than one resume beforeStart action'.format( + case['scenarioId'] + ) + ) + + action = candidate + + return action + + +def assert_before_start_actions(test, case, storage): + action = resume_before_start_action(case) + if action is None: + return + + test.assertIsNotNone(storage, case['scenarioId']) + test.assertIsNotNone(case['storedUpload'], case['scenarioId']) + test.assertEqual(action['selectedPreviousUploadIndex'], 0, case['scenarioId']) + fingerprint = case['storedUpload']['fingerprint'] + stored_upload_count = 1 if storage.get_item(fingerprint) is not None else 0 + test.assertEqual( + stored_upload_count, + action['expectedPreviousUploadCount'], + case['scenarioId'], + ) + + +def assert_events(test, case, events): + expected_events = case['eventKeys'] + extra_prefixes = case['eventKeyExtraPrefixes'] + event_policy = case['eventPolicy'] + matching = event_policy['matching'] + + if matching == 'exact': + test.assertEqual(events, expected_events, case['scenarioId']) + return + + if matching == 'exact-except-allowed-extra-events': + expected_index = 0 + for event in events: + if ( + expected_index < len(expected_events) + and event == expected_events[expected_index] + ): + expected_index += 1 + continue + + test.assertTrue( + has_allowed_extra_event_prefix(event, extra_prefixes), + '{} emitted an unexpected extra event {}; allowed prefixes {}; expected {}'.format( + case['scenarioId'], event, extra_prefixes, expected_events + ), + ) + + test.assertEqual( + expected_index, + len(expected_events), + '{} did not emit every expected non-extra event; observed {}; expected {}'.format( + case['scenarioId'], events, expected_events + ), + ) + return + + raise AssertionError( + '{} uses unsupported generated event policy {}'.format( + case['scenarioId'], event_policy + ) + ) + + +class GeneratedTusRuntimeEventsTest(unittest.TestCase): + @responses.activate + def test_sync_uploader_emits_generated_progress_and_chunk_events(self): + for case in CASES: + events = [] + client = TusClient(case['endpointUrl']) + storage = storage_for(case) + resume_action = resume_before_start_action(case) + first_call_index = len(responses.calls) + + for request in case['requests']: + url = case['endpointUrl'] if request['url'] == 'endpoint' else case['uploadUrl'] + responses.add( + request['method'], + url, + adding_headers=response_headers_for(request), + status=request['statusCode'], + ) + + assert_before_start_actions(self, case, storage) + uploader = client.uploader( + file_stream=io.BytesIO(case['content'].encode('utf-8')), + chunk_size=case['chunkSize'], + metadata=case['metadata'], + store_url=resume_action is not None, + url_storage=storage, + fingerprinter=fingerprinter_for(case, resume_action), + remove_fingerprint_on_success=case['removeFingerprintOnSuccess'], + upload_length_deferred=case['uploadLengthDeferred'], + on_progress=record_progress(events), + on_chunk_complete=record_chunk_complete(events), + ) + uploader.upload() + + assert_events(self, case, events) + assert_request_sequence(self, case, responses.calls[first_call_index:]) + assert_stored_upload_state(self, case, storage) + + +def storage_for(case): + if case['storedUpload'] is None: + return None + + return GeneratedTusStorage({ + case['storedUpload']['fingerprint']: case['storedUpload']['uploadUrl'], + }) + + +def fingerprinter_for(case, resume_action): + if resume_action is None: + return None + + if case['storedUpload'] is None: + raise AssertionError( + '{} cannot resume without a generated stored upload'.format( + case['scenarioId'] + ) + ) + + return GeneratedTusFingerprinter(case['storedUpload']['fingerprint']) + + +def request_header(request, name): + return request.headers.get(name) or request.headers.get(name.lower()) + + +def response_headers_for(request): + headers = {} + if request['includesDefaultProtocolResponseHeaders']: + headers.update(DEFAULT_RESPONSE_HEADERS) + headers.update(request['responseHeaders']) + return headers + + +def assert_request_sequence(test, case, calls): + test.assertEqual(len(calls), len(case['requests']), case['scenarioId']) + + for index, expected_request in enumerate(case['requests']): + actual_request = calls[index].request + expected_url = ( + case['endpointUrl'] + if expected_request['url'] == 'endpoint' + else case['uploadUrl'] + ) + + test.assertEqual(actual_request.method, expected_request['method'], case['scenarioId']) + test.assertEqual(actual_request.url, expected_url, case['scenarioId']) + if expected_request['includesDefaultProtocolRequestHeaders']: + for name, value in DEFAULT_REQUEST_HEADERS.items(): + test.assertEqual(request_header(actual_request, name), value, case['scenarioId']) + for name, value in expected_request['headers'].items(): + test.assertEqual(request_header(actual_request, name), value, case['scenarioId']) + + +def assert_stored_upload_state(test, case, storage): + if case['storedUpload'] is None: + return + + fingerprint = case['storedUpload']['fingerprint'] + if should_remove_stored_upload_on_success(case): + test.assertIsNone(storage.get_item(fingerprint), case['scenarioId']) + else: + test.assertEqual( + storage.get_item(fingerprint), + case['storedUpload']['uploadUrl'], + case['scenarioId'], + ) + + +def should_remove_stored_upload_on_success(case): + if not GENERATED_TUS_SUCCESS_REMOVE_STORED_URL_BEFORE_HOOK: + return False + if not GENERATED_TUS_URL_STORAGE_REMOVE_ON_SUCCESS_ENABLED: + return False + if ( + GENERATED_TUS_SUCCESS_REMOVE_STORED_URL_REQUIRES_OPTION + or GENERATED_TUS_URL_STORAGE_REMOVE_ON_SUCCESS_REQUIRES_OPTION + ): + return case['removeFingerprintOnSuccess'] + + return True diff --git a/tests/test_request.py b/tests/test_request.py index ac517e6..3372b0b 100644 --- a/tests/test_request.py +++ b/tests/test_request.py @@ -3,8 +3,11 @@ from parametrize import parametrize import responses +import requests +from tusclient.exceptions import TusUploadAborted from tusclient import request +from tusclient.request_lifecycle import RequestLifecycleHooks from tests import mixin @@ -31,6 +34,41 @@ def test_perform(self, filename: str): self.request.perform() self.assertEqual(str(size), self.request.response_headers['upload-offset']) + def test_perform_request_lifecycle_hooks(self): + events = [] + + def before_request(context): + events.append(("before", context.method, context.url)) + context.headers["x-hook"] = "before" + + def after_response(context, response): + events.append(("after", context.method, response.status_code)) + response.headers["upload-offset"] = "7" + + def validate_headers(req): + self.assertEqual(req.headers["x-hook"], "before") + return (204, {"upload-offset": "5"}, "") + + self.client.set_request_hooks( + RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ) + ) + + with responses.RequestsMock() as resps: + resps.add_callback(responses.PATCH, self.url, callback=validate_headers) + self.request.perform() + + self.assertEqual(self.request.response_headers["upload-offset"], "7") + self.assertEqual( + events, + [ + ("before", "PATCH", self.url), + ("after", "PATCH", 204), + ], + ) + def test_perform_checksum(self): self.uploader.upload_checksum = True tus_request = request.TusRequest(self.uploader) @@ -67,3 +105,14 @@ def validate_verify(req): tus_request.perform() self.assertEqual(verify, False) + def test_perform_maps_aborted_transport_error(self): + tus_request = request.TusRequest(self.uploader) + self.uploader.abort() + + def disconnect(req): + raise requests.exceptions.ConnectionError("connection closed") + + with responses.RequestsMock() as resps: + resps.add_callback(responses.PATCH, self.url, callback=disconnect) + with self.assertRaises(TusUploadAborted): + tus_request.perform() diff --git a/tests/test_start_validation.py b/tests/test_start_validation.py new file mode 100644 index 0000000..4a58527 --- /dev/null +++ b/tests/test_start_validation.py @@ -0,0 +1,43 @@ +from io import BytesIO + +import pytest +import responses + +from tusclient import client +from tusclient.start_validation import validate_upload_start + + +def test_start_validation_uses_generated_message_for_parallel_creation_upload(): + tus_client = client.TusClient("http://tusd.tusdemo.net/files/") + + validation = validate_upload_start( + client=tus_client, + file_stream=BytesIO(b"hello world"), + parallel_uploads=2, + upload_data_during_creation=True, + ) + + assert validation == { + "message": "tus: cannot use the `uploadDataDuringCreation` option when parallelUploads is enabled", + "ok": False, + "reason": "parallelUploadsWithUploadDataDuringCreation", + } + + +@responses.activate +def test_parallel_uploads_with_upload_url_fails_before_offset_request(): + tus_client = client.TusClient("http://tusd.tusdemo.net/files/") + upload_url = "http://tusd.tusdemo.net/files/15acd89eabdf5738ffc" + responses.add(responses.HEAD, upload_url, adding_headers={"upload-offset": "0"}) + + with pytest.raises(ValueError) as error: + tus_client.uploader( + file_stream=BytesIO(b"hello world"), + parallel_uploads=2, + url=upload_url, + ) + + assert str(error.value) == ( + "tus: cannot use the `uploadUrl` option when parallelUploads is enabled" + ) + assert len(responses.calls) == 0 diff --git a/tests/test_uploader.py b/tests/test_uploader.py index 9196c06..acdb7d4 100644 --- a/tests/test_uploader.py +++ b/tests/test_uploader.py @@ -10,6 +10,9 @@ import pytest from tusclient import exceptions +from tusclient.fingerprint import fingerprint +from tusclient.protocol_generated import DEFAULT_REQUEST_HEADERS, REQUEST_ID_HEADER_NAME +from tusclient.request_lifecycle import RequestLifecycleHooks from tusclient.storage import filestorage from tests import mixin @@ -34,10 +37,35 @@ def test_instance_attributes(self): self.assertEqual(self.uploader.offset, 0) def test_headers(self): - self.assertEqual(self.uploader.get_headers(), {"Tus-Resumable": "1.0.0"}) + self.assertEqual(self.uploader.get_headers(), DEFAULT_REQUEST_HEADERS) self.client.set_headers({'foo': 'bar'}) - self.assertEqual(self.uploader.get_headers(), {"Tus-Resumable": "1.0.0", 'foo': 'bar'}) + self.assertEqual(self.uploader.get_headers(), dict(DEFAULT_REQUEST_HEADERS, foo='bar')) + + self.client.set_headers({REQUEST_ID_HEADER_NAME: 'custom-request-id'}) + self.client.enable_request_id_header() + request_headers = self.uploader.get_headers() + self.assertEqual(request_headers['foo'], 'bar') + self.assertNotEqual(request_headers[REQUEST_ID_HEADER_NAME], 'custom-request-id') + self.assertEqual(len(request_headers[REQUEST_ID_HEADER_NAME]), 36) + self.assertIn('-', request_headers[REQUEST_ID_HEADER_NAME]) + + def test_prepare_request_headers_applies_operation_headers_before_custom_headers(self): + self.client.set_headers({'upload-offset': 'custom-offset'}) + request_headers = self.uploader.prepare_request_headers({'upload-offset': '1'}) + self.assertEqual(request_headers['upload-offset'], 'custom-offset') + + def test_prepare_request_headers_applies_request_id_after_custom_headers(self): + self.client.set_headers({REQUEST_ID_HEADER_NAME: 'custom-request-id'}) + self.client.enable_request_id_header() + request_headers = self.uploader.prepare_request_headers( + {REQUEST_ID_HEADER_NAME: 'operation-request-id'} + ) + + self.assertNotEqual(request_headers[REQUEST_ID_HEADER_NAME], 'operation-request-id') + self.assertNotEqual(request_headers[REQUEST_ID_HEADER_NAME], 'custom-request-id') + self.assertEqual(len(request_headers[REQUEST_ID_HEADER_NAME]), 36) + self.assertIn('-', request_headers[REQUEST_ID_HEADER_NAME]) @responses.activate def test_get_offset(self): @@ -45,6 +73,43 @@ def test_get_offset(self): adding_headers={"upload-offset": "300"}) self.assertEqual(self.uploader.get_offset(), 300) + @responses.activate + def test_get_offset_request_lifecycle_hooks(self): + events = [] + + def before_request(context): + events.append(("before", context.method, context.url)) + context.headers["x-hook"] = "before" + + def after_response(context, response): + events.append(("after", context.method, response.status_code)) + response.headers["upload-offset"] = "301" + + def validate_headers(req): + self.assertEqual(req.headers["x-hook"], "before") + return (200, {"upload-offset": "300"}, "") + + self.client.set_request_hooks( + RequestLifecycleHooks( + before_request=before_request, + after_response=after_response, + ) + ) + responses.add_callback( + responses.HEAD, + self.uploader.url, + callback=validate_headers, + ) + + self.assertEqual(self.uploader.get_offset(), 301) + self.assertEqual( + events, + [ + ("before", "HEAD", self.uploader.url), + ("after", "HEAD", 200), + ], + ) + def test_encode_metadata(self): self.uploader.metadata = {'foo': 'bar', 'red': 'blue'} encoded_metadata = ['foo' + ' ' + b64encode(b'bar').decode('ascii'), @@ -85,9 +150,16 @@ def test_url(self, filename: str): # test for stored urls responses.add(responses.HEAD, 'http://tusd.tusdemo.net/files/foo_bar', adding_headers={"upload-offset": "10"}) - storage_path = '{}/storage_file'.format(os.path.dirname(os.path.abspath(__file__))) + temp_fp = tempfile.NamedTemporaryFile(delete=False) + temp_fp.close() + storage = filestorage.FileStorage(temp_fp.name) + self.addCleanup(lambda: os.path.exists(temp_fp.name) and os.remove(temp_fp.name)) + self.addCleanup(storage.close) + with open(filename, "rb") as stream: + key = fingerprint.Fingerprint().get_fingerprint(stream) + storage.set_item(key, "http://tusd.tusdemo.net/files/foo_bar") resumable_uploader = self.client.uploader( - file_path=filename, store_url=True, url_storage=filestorage.FileStorage(storage_path) + file_path=filename, store_url=True, url_storage=storage ) self.assertEqual(resumable_uploader.url, "http://tusd.tusdemo.net/files/foo_bar") self.assertEqual(resumable_uploader.offset, 10) @@ -211,6 +283,107 @@ def test_upload(self, request_mock): self.uploader.upload() self.assertEqual(self.uploader.offset, self.uploader.get_file_size()) + @responses.activate + def test_parallel_upload_concat(self): + content = b"hello world" + part_1_url = f"{self.client.url}parallel-part-1" + part_2_url = f"{self.client.url}parallel-part-2" + final_url = f"{self.client.url}parallel-final" + events = [] + + def add_expected_request(method, url, expected_headers, response_headers, body=b""): + def callback(request): + for name, value in expected_headers.items(): + self.assertEqual(request.headers.get(name), value) + self.assertEqual(request.body or b"", body) + return (201 if method == responses.POST else 204, response_headers, "") + + responses.add_callback(method, url, callback=callback) + + add_expected_request( + responses.POST, + self.client.url, + { + "Tus-Resumable": "1.0.0", + "Upload-Concat": "partial", + "Upload-Length": "5", + "Upload-Metadata": "test d29ybGQ=", + }, + {"Location": part_1_url}, + ) + add_expected_request( + responses.POST, + self.client.url, + { + "Tus-Resumable": "1.0.0", + "Upload-Concat": "partial", + "Upload-Length": "6", + "Upload-Metadata": "test d29ybGQ=", + }, + {"Location": part_2_url}, + ) + add_expected_request( + responses.PATCH, + part_1_url, + { + "Content-Type": "application/offset+octet-stream", + "Tus-Resumable": "1.0.0", + "Upload-Offset": "0", + }, + {"Upload-Offset": "5"}, + body=b"hello", + ) + add_expected_request( + responses.PATCH, + part_2_url, + { + "Content-Type": "application/offset+octet-stream", + "Tus-Resumable": "1.0.0", + "Upload-Offset": "0", + }, + {"Upload-Offset": "6"}, + body=b" world", + ) + + def final_callback(request): + self.assertEqual(request.headers.get("Tus-Resumable"), "1.0.0") + self.assertEqual( + request.headers.get("Upload-Concat"), + "final;{} {}".format(part_1_url, part_2_url), + ) + self.assertEqual(request.headers.get("Upload-Metadata"), "foo aGVsbG8=") + self.assertNotIn("Upload-Length", request.headers) + self.assertEqual(request.body or b"", b"") + return (201, {"Location": final_url}, "") + + responses.add_callback(responses.POST, self.client.url, callback=final_callback) + + uploader = self.client.uploader( + file_stream=io.BytesIO(content), + metadata={"foo": "hello"}, + metadata_for_partial_uploads={"test": "world"}, + parallel_uploads=2, + on_progress=lambda bytes_sent, bytes_total: events.append( + ("progress", bytes_sent, bytes_total), + ), + on_chunk_complete=lambda chunk_size, bytes_accepted, bytes_total: events.append( + ("chunk-complete", chunk_size, bytes_accepted, bytes_total), + ), + ) + uploader.upload() + + self.assertEqual(uploader.url, final_url) + self.assertEqual(uploader.offset, len(content)) + self.assertEqual( + events, + [ + ("progress", 5, 11), + ("chunk-complete", 5, 5, 11), + ("progress", 11, 11), + ("chunk-complete", 6, 11, 11), + ], + ) + @mock.patch('tusclient.uploader.uploader.TusRequest') def test_upload_retry(self, request_mock): num_of_retries = 3 @@ -225,6 +398,74 @@ def test_upload_retry(self, request_mock): self.uploader.upload_chunk() self.assertEqual(self.uploader._retried, num_of_retries) + @mock.patch('tusclient.uploader.uploader.TusRequest') + def test_upload_retry_delays_and_should_retry_reset_after_progress(self, request_mock): + first_failure = mock.Mock() + first_failure.status_code = 500 + first_failure.response_content = b'' + first_failure.response_headers = {} + first_failure.perform.return_value = None + + second_failure = mock.Mock() + second_failure.status_code = 500 + second_failure.response_content = b'' + second_failure.response_headers = {} + second_failure.perform.return_value = None + + success = mock.Mock() + success.status_code = 204 + success.response_content = b'' + success.response_headers = {'upload-offset': '11'} + success.perform.return_value = None + + retry_attempts = [] + + def should_retry(error, retry_attempt): + retry_attempts.append(retry_attempt) + return True + + self.uploader.retries = 0 + self.uploader.retry_delays = [250] + self.uploader.on_should_retry = should_retry + self.uploader.offset = 0 + request_mock.side_effect = [first_failure, second_failure, success] + + with mock.patch.object(self.uploader, 'get_offset', side_effect=[5, 5]) as get_offset: + with mock.patch('tusclient.upload_chunks_generated.time.sleep') as sleep: + self.uploader.upload_chunk() + + self.assertEqual(retry_attempts, [0, 0]) + self.assertEqual(self.uploader.offset, 11) + self.assertEqual(get_offset.call_count, 2) + sleep.assert_has_calls([mock.call(0.25), mock.call(0.25)]) + + @mock.patch('tusclient.uploader.uploader.TusRequest') + def test_upload_retry_stops_when_should_retry_returns_false(self, request_mock): + failure = mock.Mock() + failure.status_code = 500 + failure.response_content = b'' + failure.response_headers = {} + failure.perform.return_value = None + + retry_attempts = [] + + def should_retry(error, retry_attempt): + retry_attempts.append(retry_attempt) + return False + + self.uploader.retries = 0 + self.uploader.retry_delays = [0] + self.uploader.on_should_retry = should_retry + request_mock.side_effect = [failure] + + with mock.patch.object(self.uploader, 'get_offset') as get_offset: + with pytest.raises(exceptions.TusCommunicationError): + self.uploader.upload_chunk() + + self.assertEqual(retry_attempts, [0]) + self.assertEqual(self.uploader._retried, 0) + get_offset.assert_not_called() + @responses.activate def test_upload_empty(self): responses.add( diff --git a/tusclient/abort_generated.py b/tusclient/abort_generated.py new file mode 100644 index 0000000..854cff8 --- /dev/null +++ b/tusclient/abort_generated.py @@ -0,0 +1,82 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +import time + +from tusclient.exceptions import TusCommunicationError + + +DEFAULT_RETRY_DELAYS = [ + 0, + 1000, + 3000, + 5000, +] +RETRY_ATTEMPT_INCREMENT_POLICY = 'after-retry-scheduled' +RETRY_CLIENT_ERROR_STATUS_CATEGORY = 400 +RETRYABLE_CLIENT_STATUS_CODES = [ + 409, + 423, +] + + +def normalized_retry_delays(retry_delays): + if retry_delays is None: + return list(DEFAULT_RETRY_DELAYS) + return list(retry_delays) + + +def should_retry_status(status_code): + if not status_code: + return False + is_client_error = ( + status_code >= RETRY_CLIENT_ERROR_STATUS_CATEGORY + and status_code < RETRY_CLIENT_ERROR_STATUS_CATEGORY + 100 + ) + return not is_client_error or status_code in RETRYABLE_CLIENT_STATUS_CODES + + +def should_schedule_retry(on_should_retry, error, retry_attempt, retry_delays): + if retry_attempt >= len(retry_delays) or not should_retry_status(error.status_code): + return False + if on_should_retry is not None: + return bool(on_should_retry(error, retry_attempt)) + return True + + +def next_retry_attempt(retry_attempt): + if RETRY_ATTEMPT_INCREMENT_POLICY == 'after-retry-scheduled': + return retry_attempt + 1 + raise ValueError( + 'tus: unsupported retry increment policy {}'.format( + RETRY_ATTEMPT_INCREMENT_POLICY, + ) + ) + + +def terminate_upload_with_retry( + upload_url, + send_terminate_request, + retry_delays=None, + on_should_retry=None, +): + retry_delays = normalized_retry_delays(retry_delays) + retry_attempt = 0 + + while True: + error = None + try: + response = send_terminate_request(upload_url) + except TusCommunicationError as terminate_error: + error = terminate_error + if error is None: + return response + + if not should_schedule_retry(on_should_retry, error, retry_attempt, retry_delays): + raise error + + delay_ms = retry_delays[retry_attempt] + if delay_ms > 0: + time.sleep(delay_ms / 1000) + retry_attempt = next_retry_attempt(retry_attempt) diff --git a/tusclient/async_upload_chunks_generated.py b/tusclient/async_upload_chunks_generated.py new file mode 100644 index 0000000..ebd8967 --- /dev/null +++ b/tusclient/async_upload_chunks_generated.py @@ -0,0 +1,65 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +import asyncio + +from tusclient.abort_generated import next_retry_attempt +from tusclient.exceptions import TusCommunicationError, TusUploadFailed +from tusclient.upload_chunks_generated import ( + effective_retry_attempt, + should_schedule_upload_retry, +) + + +async def upload_chunk_with_retry( + uploader, + perform_patch_request, + retry_delays, + on_should_retry=None, +): + uploader._retried = 0 + offset_before_retry = uploader.offset + start_offset = uploader.offset + + if not uploader.upload_length_deferred: + uploader.notify_progress(start_offset) + + while True: + error = None + try: + await perform_patch_request() + except TusUploadFailed as patch_error: + error = patch_error + if error is None: + if uploader.upload_length_deferred: + uploader.notify_progress(start_offset) + uploader.notify_progress(uploader.offset) + uploader.notify_chunk_complete(uploader.offset - start_offset, uploader.offset) + return + + while error is not None: + effective_attempt = effective_retry_attempt( + uploader._retried, + uploader.offset, + offset_before_retry, + ) + if not should_schedule_upload_retry( + on_should_retry, + error, + effective_attempt, + retry_delays, + ): + raise error + + delay_ms = retry_delays[effective_attempt] + if delay_ms > 0: + await asyncio.sleep(delay_ms / 1000) + uploader._retried = next_retry_attempt(effective_attempt) + offset_before_retry = uploader.offset + error = None + try: + uploader.offset = uploader.get_offset() + except TusCommunicationError as sync_error: + error = sync_error + continue diff --git a/tusclient/client.py b/tusclient/client.py index f878d77..5525f4c 100644 --- a/tusclient/client.py +++ b/tusclient/client.py @@ -1,5 +1,17 @@ from typing import Dict, Optional, Tuple, Union +import requests + +from tusclient.abort_generated import terminate_upload_with_retry +from tusclient.exceptions import TusCommunicationError +from tusclient.protocol_generated import ( + ABORT_REMOVE_STORED_URL_AFTER_TERMINATION, + TERMINATE_UPLOAD_METHOD, + is_successful_response_status, + prepare_request_headers, +) +from tusclient.request_lifecycle import RequestLifecycleHooks +from tusclient.request_lifecycle import TusRequestContext from tusclient.uploader import Uploader, AsyncUploader @@ -27,10 +39,21 @@ class TusClient: - client_cert (Optional[str | Tuple[str, str]]) """ - def __init__(self, url: str, headers: Optional[Dict[str, str]] = None, client_cert: Optional[Union[str, Tuple[str, str]]] = None): + def __init__( + self, + url: str, + headers: Optional[Dict[str, str]] = None, + client_cert: Optional[Union[str, Tuple[str, str]]] = None, + request_hooks: Optional[RequestLifecycleHooks] = None, + add_request_id: bool = False, + ): self.url = url self.headers = headers or {} self.client_cert = client_cert + self.request_hooks = request_hooks + self.add_request_id = add_request_id + self._abort_requested = False + self._current_uploader = None def set_headers(self, headers: Dict[str, str]): """ @@ -45,6 +68,95 @@ def set_headers(self, headers: Dict[str, str]): """ self.headers.update(headers) + def set_request_hooks(self, request_hooks: Optional[RequestLifecycleHooks]): + """ + Set callbacks that are invoked around each HTTP request/response pair. + + :Args: + - request_hooks (Optional[RequestLifecycleHooks]): + callbacks to invoke before transport send and after transport response. + """ + self.request_hooks = request_hooks + + def enable_request_id_header(self): + self.add_request_id = True + + def disable_request_id_header(self): + self.add_request_id = False + + def _set_current_uploader(self, uploader): + self._current_uploader = uploader + if self._abort_requested: + self._abort_requested = False + uploader.abort() + + def _clear_current_uploader(self, uploader): + if self._current_uploader is uploader: + self._current_uploader = None + + def abort_upload( + self, + uploader=None, + terminate_upload: bool = False, + verify_tls_cert: bool = True, + ): + active_uploader = uploader or self._current_uploader + if active_uploader is None: + self._abort_requested = True + return None + + active_uploader.abort() + if not terminate_upload or not active_uploader.url: + return None + + response = self.terminate_upload( + active_uploader.url, + verify_tls_cert=verify_tls_cert, + ) + if ABORT_REMOVE_STORED_URL_AFTER_TERMINATION == "after-successful-termination": + active_uploader.remove_stored_url() + return response + + def terminate_upload(self, upload_url: str, verify_tls_cert: bool = True): + # The retry/abort sequencing lives in the generated termination runtime; this client + # keeps its historical single-attempt behavior by passing an empty retry budget. + return terminate_upload_with_retry( + upload_url, + lambda url: self._send_terminate_request(url, verify_tls_cert), + retry_delays=[], + ) + + def _send_terminate_request(self, upload_url: str, verify_tls_cert: bool): + headers = prepare_request_headers(None, self.headers, self.add_request_id) + context = TusRequestContext(TERMINATE_UPLOAD_METHOD, upload_url, headers) + if self.request_hooks is not None and self.request_hooks.before_request is not None: + self.request_hooks.before_request(context) + + try: + response = requests.request( + TERMINATE_UPLOAD_METHOD, + upload_url, + headers=context.headers, + verify=verify_tls_cert, + cert=self.client_cert, + ) + except requests.exceptions.RequestException as error: + raise TusCommunicationError(error) + + if self.request_hooks is not None and self.request_hooks.after_response is not None: + self.request_hooks.after_response(context, response) + + if not is_successful_response_status(response.status_code): + raise TusCommunicationError( + "unexpected status code ({}) while terminating upload".format( + response.status_code + ), + response.status_code, + response.content, + ) + + return response + def uploader(self, *args, **kwargs) -> Uploader: """ Return uploader instance pointing at current client instance. @@ -58,6 +170,20 @@ def uploader(self, *args, **kwargs) -> Uploader: kwargs["client"] = self return Uploader(*args, **kwargs) + def create_upload_with_data(self, bytes_to_upload: int, *args, **kwargs) -> Uploader: + """ + Create an upload URL while sending the first bytes in the creation request. + + :Args: + - bytes_to_upload (int): + Number of bytes to send during upload creation. + see tusclient.uploader.Uploader for remaining arguments. + """ + kwargs["upload_data_during_creation"] = True + uploader = self.uploader(*args, **kwargs) + uploader.create_url_with_upload(bytes_to_upload) + return uploader + def async_uploader(self, *args, **kwargs) -> AsyncUploader: kwargs["client"] = self return AsyncUploader(*args, **kwargs) diff --git a/tusclient/detailed_error.py b/tusclient/detailed_error.py new file mode 100644 index 0000000..d40fcbd --- /dev/null +++ b/tusclient/detailed_error.py @@ -0,0 +1,106 @@ +from tusclient.exceptions import TusDetailedError +from tusclient.protocol_generated import ( + DETAILED_ERROR_CAUSE_STRING_TEMPLATE, + DETAILED_ERROR_CAUSED_BY_TEMPLATE, + DETAILED_ERROR_CREATE_UPLOAD_REQUEST_FAILED, + DETAILED_ERROR_EMPTY_RESPONSE_BODY, + DETAILED_ERROR_MISSING_VALUE, + DETAILED_ERROR_REQUEST_CONTEXT_TEMPLATE, + DETAILED_ERROR_UNEXPECTED_CREATE_RESPONSE, + REQUEST_ID_HEADER_NAME, +) + + +def create_upload_response_error(context, response): + body = response_body(response) + message = detailed_error_message( + DETAILED_ERROR_UNEXPECTED_CREATE_RESPONSE, + request_context={ + 'body': body, + 'method': context.method, + 'requestId': request_id(context), + 'status': response.status_code, + 'url': context.url, + }, + ) + return TusDetailedError( + message, + status_code=response.status_code, + response_content=response.content, + original_request_method=context.method, + original_request_url=context.url, + original_request_id=request_id(context), + original_response_body=body, + original_response_present=True, + original_response_status=response.status_code, + ) + + +def create_upload_request_error(context, error): + message = detailed_error_message( + DETAILED_ERROR_CREATE_UPLOAD_REQUEST_FAILED, + cause=error, + request_context={ + 'body': DETAILED_ERROR_MISSING_VALUE, + 'method': context.method, + 'requestId': request_id(context), + 'status': DETAILED_ERROR_MISSING_VALUE, + 'url': context.url, + }, + ) + return TusDetailedError( + message, + causing_error=error, + original_request_method=context.method, + original_request_url=context.url, + original_request_id=request_id(context), + ) + + +def detailed_error_message(base_message, cause=None, request_context=None): + message = base_message + if cause is not None: + cause_message = format_flow_message( + DETAILED_ERROR_CAUSE_STRING_TEMPLATE, + {'message': missing_if_empty(str(cause))}, + ) + message += format_flow_message( + DETAILED_ERROR_CAUSED_BY_TEMPLATE, + {'cause': cause_message}, + ) + + if request_context is not None: + message += format_flow_message( + DETAILED_ERROR_REQUEST_CONTEXT_TEMPLATE, + request_context, + ) + + return message + + +def format_flow_message(template, values): + message = template + for key, value in values.items(): + message = message.replace('{{{}}}'.format(key), str(value)) + return message + + +def missing_if_empty(value): + if value is None or value == '': + return DETAILED_ERROR_MISSING_VALUE + return value + + +def request_id(context): + return missing_if_empty(context.headers.get(REQUEST_ID_HEADER_NAME)) + + +def response_body(response): + body = response.content + if body is None: + return DETAILED_ERROR_MISSING_VALUE + if body == b'': + return DETAILED_ERROR_EMPTY_RESPONSE_BODY + if isinstance(body, bytes): + return body.decode('utf-8', 'replace') + return str(body) diff --git a/tusclient/exceptions.py b/tusclient/exceptions.py index 959ac31..205fe7a 100644 --- a/tusclient/exceptions.py +++ b/tusclient/exceptions.py @@ -2,6 +2,8 @@ Global Tusclient exception and warning classes. """ +from tusclient.protocol_generated import ABORT_ERROR_MESSAGE + class TusCommunicationError(Exception): """ @@ -31,5 +33,42 @@ def __init__(self, message, status_code=None, response_content=None): self.response_content = response_content +class TusDetailedError(TusCommunicationError): + """Communication error that preserves original request and response context.""" + + def __init__( + self, + message, + status_code=None, + response_content=None, + causing_error=None, + original_request_method=None, + original_request_url=None, + original_request_id=None, + original_response_body=None, + original_response_present=False, + original_response_status=None, + ): + super(TusDetailedError, self).__init__( + message, + status_code=status_code, + response_content=response_content, + ) + self.causing_error = causing_error + self.original_request_method = original_request_method + self.original_request_url = original_request_url + self.original_request_id = original_request_id + self.original_response_body = original_response_body + self.original_response_present = original_response_present + self.original_response_status = original_response_status + + class TusUploadFailed(TusCommunicationError): """Should be raised when an attempted upload fails""" + + +class TusUploadAborted(TusCommunicationError): + """Should be raised when an upload request is explicitly aborted""" + + def __init__(self, message=ABORT_ERROR_MESSAGE): + super(TusUploadAborted, self).__init__(message) diff --git a/tusclient/protocol_generated.py b/tusclient/protocol_generated.py new file mode 100644 index 0000000..96b8c4f --- /dev/null +++ b/tusclient/protocol_generated.py @@ -0,0 +1,496 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +ABORT_ERROR_MESSAGE = 'Request was aborted' +ABORT_ERROR_TYPE = 'DOMException' +ABORT_REMOVE_STORED_URL_AFTER_TERMINATION = 'after-successful-termination' +ABORT_SEQUENCE = [ + 'mark-aborted', + 'abort-parallel-uploads', + 'abort-current-request', + 'clear-retry-timer', + 'terminate-upload-if-requested', +] +ABORT_SUPPRESS_ERROR_AFTER_ABORT = True +ABORT_TERMINATE_UPLOAD = 'when-requested-and-upload-url-known' +ABORT_TERMINATE_UPLOAD_CONTEXT = 'detached-from-aborted-request' +CREATE_UPLOAD_METHOD = 'POST' +DEFAULT_CLIENT_PROTOCOL = 'tus-v1' +DEFAULT_PROTOCOL_VERSION = '1.0.0' +DEFAULT_REQUEST_HEADERS = { + 'Tus-Resumable': '1.0.0', +} +DEFAULT_RESPONSE_HEADERS = { + 'Tus-Resumable': '1.0.0', +} +DETAILED_ERROR_CAUSE_STRING_TEMPLATE = 'Error: {message}' +DETAILED_ERROR_CAUSED_BY_TEMPLATE = ', caused by {cause}' +DETAILED_ERROR_CREATE_UPLOAD_REQUEST_FAILED = 'tus: failed to create upload' +DETAILED_ERROR_EMPTY_RESPONSE_BODY = '' +DETAILED_ERROR_MISSING_VALUE = 'n/a' +DETAILED_ERROR_REQUEST_CONTEXT_TEMPLATE = ', originated from request (method: {method}, url: {url}, response code: {status}, response text: {body}, request id: {requestId})' +DETAILED_ERROR_UNEXPECTED_CREATE_RESPONSE = 'tus: unexpected response while creating upload' +LOCATION_HEADER_NAME = 'Location' +METADATA_HEADER_NAME = 'Upload-Metadata' +METHOD_OVERRIDE_INPUT_OPTION_NAMES = { + 'overridePatchMethod': 'override_patch_method', +} +METHOD_OVERRIDES = [ + { + 'headerName': 'X-HTTP-Method-Override', + 'headerValue': 'PATCH', + 'inputFlag': 'overridePatchMethod', + 'method': 'POST', + 'operationId': 'patchTusUpload', + }, +] +OFFSET_DISCOVERY_METHOD = 'HEAD' +PARALLEL_FINAL_CONCAT_PREFIX = 'final;' +PARALLEL_PARTIAL_HEADER_KIND = 'partial-upload' +PARALLEL_PARTIAL_METADATA_SOURCE = 'metadataForPartialUploads' +PARALLEL_PARTIAL_NESTED_UPLOADS = 'disabled' +PARALLEL_PARTIAL_URL_STORAGE = 'parent-managed' +PARALLEL_UPLOAD_DEFAULT = 1 +PARALLEL_UPLOAD_MINIMUM = 2 +PARALLEL_UPLOAD_SPLIT_STRATEGY = 'contiguous-floor-size-last-remainder' +PARALLEL_UPLOAD_URL_SEPARATOR = ' ' +REQUEST_ID_HEADER_NAME = 'X-Request-ID' +START_VALIDATION_CLIENT_FLOW_VALUES = { + 'minimumParallelUploads': 2, +} +START_VALIDATION_MESSAGES = { + 'configuredUploadSizeMismatch': 'upload was configured with a size of {expectedSize} bytes, but the source is done after {actualSize} bytes', + 'cannotDeriveUploadSize': 'tus: cannot automatically derive upload\'s size from input. Specify it manually using the `uploadSize` option or use the `uploadLengthDeferred` option', + 'createMissingEndpoint': 'tus: unable to create upload because no endpoint is provided', + 'createMissingSize': 'tus: expected _size to be set', + 'createUploadRequestFailed': 'tus: failed to create upload', + 'createdUpload': 'Created upload at {uploadUrl}', + 'finalUploadMissingPartialUrls': 'tus: Expected _parallelUploadUrls to be set', + 'finalUploadRequestFailed': 'tus: failed to concatenate parallel uploads', + 'fingerprintCalculated': 'Calculated fingerprint: {fingerprint}', + 'fingerprintUnavailable': 'tus: unable to calculate fingerprint for this input file', + 'fingerprintUnavailableForStorage': 'No fingerprint was calculated meaning that the upload cannot be stored in the URL storage.', + 'invalidUploadSize': 'tus: cannot convert `uploadSize` option into a number', + 'invalidChunkOffset': 'tus: invalid or missing offset value', + 'invalidResumeLength': 'tus: invalid or missing length value', + 'invalidResumeOffset': 'tus: invalid Upload-Offset header', + 'lockedUpload': 'tus: upload is currently locked; retry later', + 'nonErrorThrownValue': 'tus: value thrown that is not an error: {value}', + 'missingEndpointOrUploadUrl': 'tus: neither an endpoint or an upload URL is provided', + 'missingInput': 'tus: no file or stream to upload provided', + 'missingPatchUrl': 'tus: Expected url to be set', + 'missingResumeOffset': 'tus: missing Upload-Offset header', + 'removedResumeOption': 'tus: The `resume` option has been removed in tus-js-client v2. Please use the URL storage API instead.', + 'parallelBoundariesLengthMismatch': 'tus: the `parallelUploadBoundaries` must have the same length as the value of `parallelUploads`', + 'parallelBoundariesWithoutParallelUploads': 'tus: cannot use the `parallelUploadBoundaries` option when `parallelUploads` is disabled', + 'parallelUploadMissingSize': 'tus: Expected _size to be set', + 'parallelUploadsWithDeferredLength': 'tus: cannot use the `uploadLengthDeferred` option when parallelUploads is enabled', + 'parallelUploadsWithUploadDataDuringCreation': 'tus: cannot use the `uploadDataDuringCreation` option when parallelUploads is enabled', + 'parallelUploadsWithUploadSize': 'tus: cannot use the `uploadSize` option when parallelUploads is enabled', + 'parallelUploadsWithUploadUrl': 'tus: cannot use the `uploadUrl` option when parallelUploads is enabled', + 'parallelUploadSliceMissingValue': 'tus: no value returned while slicing file for parallel uploads', + 'reactNativeUriBlobFetchFailed': 'tus: cannot fetch `file.uri` as Blob, make sure the uri is correct and accessible. {error}', + 'reactNativeUriUnsupported': 'tus: file objects with `uri` property is only supported in React Native', + 'resumeUploadRequestFailed': 'tus: failed to resume upload', + 'resumeWithoutEndpoint': 'tus: unable to resume upload (new upload cannot be created without an endpoint)', + 'retryDelaysNotArray': 'tus: the `retryDelays` option must either be an array or null', + 'storageMissingParallelUploadUrls': 'tus: cannot store parallel upload because no partial upload URLs are available', + 'storageMissingUploadUrl': 'tus: cannot store upload because no upload URL is available', + 'terminateUploadRequestFailed': 'tus: failed to terminate upload', + 'unexpectedChunkResponse': 'tus: unexpected response while uploading chunk', + 'unexpectedCreateResponse': 'tus: unexpected response while creating upload', + 'unexpectedResumeResponse': 'tus: unexpected response while resuming upload', + 'unexpectedTerminateResponse': 'tus: unexpected response while terminating upload', + 'uploadChunkRequestFailed': 'tus: failed to upload chunk at offset {offset}', + 'uploadLocationMissing': 'tus: invalid or missing Location header', + 'unsupportedProtocolPrefix': 'tus: unsupported protocol ', +} +START_VALIDATION_RULES = [ + { + 'message': { + 'key': 'missingInput', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'equals': False, + 'input': 'hasFile', + 'kind': 'boolean-input', + }, + 'reason': 'missingInput', + 'expectedError': 'tus: no file or stream to upload provided', + 'scenarioId': 'startValidationMissingInput', + }, + { + 'message': { + 'input': 'protocol', + 'key': 'unsupportedProtocolPrefix', + 'kind': 'client-flow-message-with-input-suffix', + }, + 'predicate': { + 'equals': False, + 'input': 'protocol', + 'kind': 'supported-protocol', + }, + 'reason': 'unsupportedProtocol', + 'expectedError': 'tus: unsupported protocol tus-v9', + 'scenarioId': 'startValidationUnsupportedProtocol', + }, + { + 'message': { + 'key': 'missingEndpointOrUploadUrl', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'equals': False, + 'input': 'hasEndpoint', + 'kind': 'boolean-input', + }, + { + 'equals': False, + 'input': 'hasUploadUrl', + 'kind': 'boolean-input', + }, + { + 'equals': False, + 'input': 'hasCurrentUrl', + 'kind': 'boolean-input', + }, + ], + }, + 'reason': 'missingEndpointOrUploadUrl', + 'expectedError': 'tus: neither an endpoint or an upload URL is provided', + 'scenarioId': 'startValidationMissingEndpointOrUploadUrl', + }, + { + 'message': { + 'key': 'retryDelaysNotArray', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'equals': False, + 'input': 'retryDelays', + 'kind': 'array-or-null', + }, + 'reason': 'retryDelaysNotArray', + 'expectedError': 'tus: the `retryDelays` option must either be an array or null', + 'scenarioId': 'startValidationRetryDelaysNotArray', + }, + { + 'message': { + 'key': 'parallelUploadsWithUploadUrl', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploads', + 'kind': 'number-input-gte-client-flow-value', + 'value': 'minimumParallelUploads', + }, + { + 'equals': True, + 'input': 'hasUploadUrl', + 'kind': 'boolean-input', + }, + ], + }, + 'reason': 'parallelUploadsWithUploadUrl', + 'expectedError': 'tus: cannot use the `uploadUrl` option when parallelUploads is enabled', + 'scenarioId': 'startValidationParallelUploadsWithUploadUrl', + }, + { + 'message': { + 'key': 'parallelUploadsWithUploadSize', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploads', + 'kind': 'number-input-gte-client-flow-value', + 'value': 'minimumParallelUploads', + }, + { + 'equals': True, + 'input': 'hasUploadSize', + 'kind': 'boolean-input', + }, + ], + }, + 'reason': 'parallelUploadsWithUploadSize', + 'expectedError': 'tus: cannot use the `uploadSize` option when parallelUploads is enabled', + 'scenarioId': 'startValidationParallelUploadsWithUploadSize', + }, + { + 'message': { + 'key': 'parallelUploadsWithDeferredLength', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploads', + 'kind': 'number-input-gte-client-flow-value', + 'value': 'minimumParallelUploads', + }, + { + 'equals': True, + 'input': 'uploadLengthDeferred', + 'kind': 'boolean-input', + }, + ], + }, + 'reason': 'parallelUploadsWithDeferredLength', + 'expectedError': 'tus: cannot use the `uploadLengthDeferred` option when parallelUploads is enabled', + 'scenarioId': 'startValidationParallelUploadsWithDeferredLength', + }, + { + 'message': { + 'key': 'parallelUploadsWithUploadDataDuringCreation', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploads', + 'kind': 'number-input-gte-client-flow-value', + 'value': 'minimumParallelUploads', + }, + { + 'equals': True, + 'input': 'uploadDataDuringCreation', + 'kind': 'boolean-input', + }, + ], + }, + 'reason': 'parallelUploadsWithUploadDataDuringCreation', + 'expectedError': 'tus: cannot use the `uploadDataDuringCreation` option when parallelUploads is enabled', + 'scenarioId': 'startValidationParallelUploadsWithUploadDataDuringCreation', + }, + { + 'message': { + 'key': 'parallelBoundariesWithoutParallelUploads', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploadBoundariesCount', + 'kind': 'number-input-not-null', + }, + { + 'input': 'parallelUploads', + 'kind': 'number-input-lt-client-flow-value', + 'value': 'minimumParallelUploads', + }, + ], + }, + 'reason': 'parallelBoundariesWithoutParallelUploads', + 'expectedError': 'tus: cannot use the `parallelUploadBoundaries` option when `parallelUploads` is disabled', + 'scenarioId': 'startValidationParallelBoundariesWithoutParallelUploads', + }, + { + 'message': { + 'key': 'parallelBoundariesLengthMismatch', + 'kind': 'client-flow-message', + }, + 'predicate': { + 'kind': 'all', + 'predicates': [ + { + 'input': 'parallelUploadBoundariesCount', + 'kind': 'number-input-not-null', + }, + { + 'kind': 'number-input-not-equals-number-input', + 'left': 'parallelUploads', + 'right': 'parallelUploadBoundariesCount', + }, + ], + }, + 'reason': 'parallelBoundariesLengthMismatch', + 'expectedError': 'tus: the `parallelUploadBoundaries` must have the same length as the value of `parallelUploads`', + 'scenarioId': 'startValidationParallelBoundariesLengthMismatch', + }, +] +SUCCESS_RESPONSE_STATUS_CATEGORY = 200 +TERMINATE_UPLOAD_METHOD = 'DELETE' +TUS_SUPPORTED_PROTOCOLS = [ + 'tus-v1', + 'ietf-draft-03', + 'ietf-draft-05', +] +TUS_PROTOCOL_REQUEST_HEADERS = { + 'tus-v1': { + 'Tus-Resumable': '1.0.0', + }, + 'ietf-draft-03': { + 'Upload-Draft-Interop-Version': '5', + }, + 'ietf-draft-05': { + 'Upload-Draft-Interop-Version': '6', + }, +} +TUS_PROTOCOL_UPLOAD_BODY_CONTENT_TYPES = { + 'ietf-draft-05': 'application/partial-upload', + 'tus-v1': 'application/offset+octet-stream', +} +TUS_PROTOCOL_UPLOAD_COMPLETE_HEADERS = { + 'ietf-draft-03': { + 'completeValue': '?1', + 'incompleteValue': '?0', + 'name': 'Upload-Complete', + }, + 'ietf-draft-05': { + 'completeValue': '?1', + 'incompleteValue': '?0', + 'name': 'Upload-Complete', + }, +} +URL_STORAGE_ID_MULTIPLIER = 1000000000000 +URL_STORAGE_ID_STRATEGY = 'rounded-random-number' +URL_STORAGE_NAMESPACE = 'tus' +URL_STORAGE_SEPARATOR = '::' +UPLOAD_BODY_CONTENT_TYPE = 'application/offset+octet-stream' +UPLOAD_BODY_CONTENT_TYPE_HEADER_NAME = 'Content-Type' +UPLOAD_CHUNK_METHOD = 'PATCH' +UPLOAD_CHUNK_OPERATION_ID = 'patchTusUpload' +UPLOAD_CONCAT_HEADER_NAME = 'Upload-Concat' +UPLOAD_CONCAT_PARTIAL_VALUE = 'partial' +UPLOAD_DEFER_LENGTH_HEADER_NAME = 'Upload-Defer-Length' +UPLOAD_LENGTH_HEADER_NAME = 'Upload-Length' +UPLOAD_OFFSET_HEADER_NAME = 'Upload-Offset' + + +def is_successful_response_status(response_status_code): + return ( + response_status_code >= SUCCESS_RESPONSE_STATUS_CATEGORY + and response_status_code < SUCCESS_RESPONSE_STATUS_CATEGORY + 100 + ) + + +def normalize_client_protocol(protocol=None): + if protocol is None or protocol == DEFAULT_PROTOCOL_VERSION: + return DEFAULT_CLIENT_PROTOCOL + return protocol + + +def protocol_request_headers(protocol=None): + normalized_protocol = normalize_client_protocol(protocol) + headers = TUS_PROTOCOL_REQUEST_HEADERS.get(normalized_protocol) + if headers is None: + raise ValueError('tus: unsupported protocol {}'.format(protocol)) + return dict(headers) + + +def protocol_upload_body_content_type(protocol=None): + return TUS_PROTOCOL_UPLOAD_BODY_CONTENT_TYPES.get(normalize_client_protocol(protocol)) + + +def upload_body_headers(protocol=None, done=None): + headers = {} + content_type = protocol_upload_body_content_type(protocol) + if content_type: + headers[UPLOAD_BODY_CONTENT_TYPE_HEADER_NAME] = content_type + if done is not None: + upload_complete_header = TUS_PROTOCOL_UPLOAD_COMPLETE_HEADERS.get( + normalize_client_protocol(protocol) + ) + if upload_complete_header: + headers[upload_complete_header['name']] = ( + upload_complete_header['completeValue'] + if done + else upload_complete_header['incompleteValue'] + ) + return headers + + +def prepare_request_headers( + operation_headers=None, + custom_headers=None, + add_request_id=False, + protocol=None, +): + headers = {} + add_operation_request_headers(headers, operation_headers, protocol) + add_custom_request_headers(headers, custom_headers) + add_request_id_header(headers, add_request_id) + return headers + + +def request_method_plan(operation_id, source_method, input_options=None): + input_options = input_options or {} + for method_override in METHOD_OVERRIDES: + if method_override['operationId'] != operation_id: + continue + + input_flag = method_override['inputFlag'] + option_name = METHOD_OVERRIDE_INPUT_OPTION_NAMES[input_flag] + if not input_options.get(option_name, False): + continue + + return { + 'headers': { + method_override['headerName']: method_override['headerValue'], + }, + 'method': method_override['method'], + } + + return { + 'headers': {}, + 'method': source_method, + } + + +def url_storage_all_uploads_prefix(): + return '{}{}'.format(URL_STORAGE_NAMESPACE, URL_STORAGE_SEPARATOR) + + +def url_storage_fingerprint_prefix(fingerprint): + return '{}{}{}'.format( + url_storage_all_uploads_prefix(), + fingerprint, + URL_STORAGE_SEPARATOR, + ) + + +def url_storage_id(random_value): + if URL_STORAGE_ID_STRATEGY != 'rounded-random-number': + raise ValueError('tus: unsupported URL storage id strategy {}'.format(URL_STORAGE_ID_STRATEGY)) + return round(random_value * URL_STORAGE_ID_MULTIPLIER) + + +def url_storage_key(fingerprint, upload_id): + return '{}{}'.format(url_storage_fingerprint_prefix(fingerprint), upload_id) + + +def add_operation_request_headers(headers, operation_headers, protocol=None): + headers.update(protocol_request_headers(protocol)) + if operation_headers: + headers.update(operation_headers) + + +def add_custom_request_headers(headers, custom_headers): + if custom_headers: + headers.update(custom_headers) + + +def add_request_id_header(headers, add_request_id): + if not add_request_id: + return + headers[REQUEST_ID_HEADER_NAME] = generated_request_id() + + +def generated_request_id(): + import uuid + + return str(uuid.uuid4()) diff --git a/tusclient/request.py b/tusclient/request.py index 942d944..3e845f3 100644 --- a/tusclient/request.py +++ b/tusclient/request.py @@ -7,7 +7,13 @@ import aiohttp import ssl -from tusclient.exceptions import TusUploadFailed, TusCommunicationError +from tusclient.exceptions import TusUploadAborted, TusUploadFailed, TusCommunicationError +from tusclient.protocol_generated import ( + UPLOAD_CHUNK_METHOD, + UPLOAD_CHUNK_OPERATION_ID, + request_method_plan, + upload_body_headers, +) # Catches requests exceptions and throws custom tuspy errors. @@ -40,6 +46,7 @@ class BaseTusRequest: """ def __init__(self, uploader): + self.uploader = uploader self._url = uploader.url self.status_code = None self.response_headers = {} @@ -50,21 +57,19 @@ def __init__(self, uploader): self.file.seek(uploader.offset) self.client_cert = uploader.client_cert - self._request_headers = { + self._operation_headers = { "upload-offset": str(uploader.offset), - "Content-Type": "application/offset+octet-stream", } self._offset = uploader.offset self._upload_length_deferred = uploader.upload_length_deferred - self._request_headers.update(uploader.get_headers()) self._content_length = uploader.get_request_length() self._upload_checksum = uploader.upload_checksum self._checksum_algorithm = uploader.checksum_algorithm self._checksum_algorithm_name = uploader.checksum_algorithm_name - def add_checksum(self, chunk: bytes): + def add_checksum(self, headers, chunk: bytes): if self._upload_checksum: - self._request_headers["upload-checksum"] = " ".join( + headers["upload-checksum"] = " ".join( ( self._checksum_algorithm_name, base64.b64encode(self._checksum_algorithm(chunk).digest()).decode( @@ -73,6 +78,18 @@ def add_checksum(self, chunk: bytes): ) ) + def request_method_plan(self): + return request_method_plan( + UPLOAD_CHUNK_OPERATION_ID, + UPLOAD_CHUNK_METHOD, + self.uploader.request_method_input_options(), + ) + + def _is_final_chunk(self, stream_eof, chunk_size): + if self._upload_length_deferred: + return stream_eof + return self._offset + chunk_size >= self.uploader.file_size + class TusRequest(BaseTusRequest): """Class to handle async Tus upload requests""" @@ -84,26 +101,42 @@ def perform(self): try: chunk = self.file.read(self._content_length) stream_eof = len(chunk) < self._content_length - self.add_checksum(chunk) - headers = self._request_headers - if stream_eof and self._upload_length_deferred: - headers["upload-length"] = str(self._offset + len(chunk)) - resp = requests.patch( - self._url, - data=chunk, - headers=headers, - verify=self.verify_tls_cert, - stream=True, - cert=self.client_cert + operation_headers = dict(self._operation_headers) + self.add_checksum(operation_headers, chunk) + operation_headers.update( + upload_body_headers( + self.uploader.protocol, + done=self._is_final_chunk(stream_eof, len(chunk)), + ) ) + if stream_eof and self._upload_length_deferred: + operation_headers["upload-length"] = str(self._offset + len(chunk)) + method_plan = self.request_method_plan() + operation_headers.update(method_plan["headers"]) + headers = self.uploader.prepare_request_headers(operation_headers) + context = self.uploader.run_before_request(method_plan["method"], self._url, headers) + try: + resp = requests.request( + method_plan["method"], + self._url, + data=chunk, + headers=context.headers, + verify=self.verify_tls_cert, + stream=True, + cert=self.client_cert, + ) + finally: + self.uploader.clear_current_request() + self.uploader.run_after_response(context, resp) self.status_code = resp.status_code self.response_content = resp.content self.response_headers = {k.lower(): v for k, v in resp.headers.items()} self.stream_eof = stream_eof except requests.exceptions.RequestException as error: + if self.uploader.is_aborted(): + raise TusUploadAborted() raise TusUploadFailed(error) - class AsyncTusRequest(BaseTusRequest): """Class to handle async Tus upload requests""" @@ -118,24 +151,51 @@ async def perform(self): Perform actual request. """ chunk = self.file.read(self._content_length) - self.add_checksum(chunk) + stream_eof = len(chunk) < self._content_length + operation_headers = dict(self._operation_headers) + self.add_checksum(operation_headers, chunk) + operation_headers.update( + upload_body_headers( + self.uploader.protocol, + done=self._is_final_chunk(stream_eof, len(chunk)), + ) + ) try: ssl_ctx = ssl.create_default_context() - if (self.client_cert is not None): + if self.client_cert is not None: if self.client_cert is str: ssl_ctx.load_cert_chain(certfile=self.client_cert) else: - ssl_ctx.load_cert_chain(certfile=self.client_cert[0], keyfile=self.client_cert[1]) + ssl_ctx.load_cert_chain( + certfile=self.client_cert[0], keyfile=self.client_cert[1] + ) conn = aiohttp.TCPConnector(ssl=ssl_ctx) async with aiohttp.ClientSession(loop=self.io_loop, connector=conn) as session: verify_tls_cert = None if self.verify_tls_cert else False - async with session.patch( - self._url, data=chunk, headers=self._request_headers, ssl=verify_tls_cert - ) as resp: - self.status_code = resp.status - self.response_headers = { - k.lower(): v for k, v in resp.headers.items() - } - self.response_content = await resp.content.read() + method_plan = self.request_method_plan() + operation_headers.update(method_plan["headers"]) + headers = self.uploader.prepare_request_headers(operation_headers) + context = self.uploader.run_before_request( + method_plan["method"], self._url, headers + ) + try: + async with session.request( + method_plan["method"], + self._url, + data=chunk, + headers=context.headers, + ssl=verify_tls_cert, + ) as resp: + self.uploader.run_after_response(context, resp) + self.status_code = resp.status + self.response_headers = { + k.lower(): v for k, v in resp.headers.items() + } + self.response_content = await resp.content.read() + self.stream_eof = stream_eof + finally: + self.uploader.clear_current_request() except aiohttp.ClientError as error: + if self.uploader.is_aborted(): + raise TusUploadAborted() raise TusUploadFailed(error) diff --git a/tusclient/request_lifecycle.py b/tusclient/request_lifecycle.py new file mode 100644 index 0000000..dc6333c --- /dev/null +++ b/tusclient/request_lifecycle.py @@ -0,0 +1,15 @@ +class TusRequestContext: + """Request context passed to lifecycle hooks before transport sends.""" + + def __init__(self, method, url, headers): + self.method = method + self.url = url + self.headers = headers + + +class RequestLifecycleHooks: + """Callbacks invoked around each HTTP request/response pair.""" + + def __init__(self, before_request=None, after_response=None): + self.before_request = before_request + self.after_response = after_response diff --git a/tusclient/start_validation.py b/tusclient/start_validation.py new file mode 100644 index 0000000..c1f5623 --- /dev/null +++ b/tusclient/start_validation.py @@ -0,0 +1,147 @@ +from tusclient.protocol_generated import ( + START_VALIDATION_CLIENT_FLOW_VALUES, + START_VALIDATION_MESSAGES, + START_VALIDATION_RULES, + TUS_SUPPORTED_PROTOCOLS, +) + + +def validate_upload_start( + file_path=None, + file_stream=None, + client=None, + url=None, + upload_size=None, + upload_data_during_creation=False, + upload_length_deferred=False, + parallel_uploads=None, + parallel_upload_boundaries=None, + protocol=None, + retry_delays=None, +): + input_values = { + "hasCurrentUrl": url is not None, + "hasEndpoint": client is not None and getattr(client, "url", None) is not None, + "hasFile": file_path is not None or file_stream is not None, + "hasUploadSize": upload_size is not None, + "hasUploadUrl": url is not None, + "parallelUploadBoundariesCount": ( + len(parallel_upload_boundaries) + if parallel_upload_boundaries is not None + else None + ), + "parallelUploads": _default_parallel_uploads(parallel_uploads), + "protocol": _default_protocol(protocol), + "retryDelays": retry_delays, + "uploadDataDuringCreation": upload_data_during_creation, + "uploadLengthDeferred": upload_length_deferred, + } + + for rule in START_VALIDATION_RULES: + if _evaluate_predicate(rule["predicate"], input_values): + return { + "message": _resolve_message(rule["message"], input_values), + "ok": False, + "reason": rule["reason"], + } + + return {"ok": True} + + +def validate_upload_start_or_raise(**kwargs): + validation = validate_upload_start(**kwargs) + if validation["ok"]: + return validation + + raise ValueError(validation["message"]) + + +def _default_parallel_uploads(value): + if value is not None: + return value + + return START_VALIDATION_CLIENT_FLOW_VALUES["minimumParallelUploads"] - 1 + + +def _default_protocol(value): + if value is not None: + return value + + return TUS_SUPPORTED_PROTOCOLS[0] + + +def _evaluate_predicate(predicate, input_values): + kind = predicate["kind"] + + if kind == "all": + return all( + _evaluate_predicate(child, input_values) + for child in predicate.get("predicates", []) + ) + + if kind == "any": + return any( + _evaluate_predicate(child, input_values) + for child in predicate.get("predicates", []) + ) + + if kind == "array-or-null": + value = input_values[predicate["input"]] + result = value is None or isinstance(value, list) + return result == predicate["equals"] + + if kind == "boolean-input": + return bool(input_values[predicate["input"]]) == predicate["equals"] + + if kind == "not": + return not _evaluate_predicate(predicate["predicate"], input_values) + + if kind == "number-input-gte-client-flow-value": + return _number_input(input_values, predicate["input"]) >= _client_flow_value( + predicate["value"] + ) + + if kind == "number-input-lt-client-flow-value": + return _number_input(input_values, predicate["input"]) < _client_flow_value( + predicate["value"] + ) + + if kind == "number-input-not-equals-number-input": + return _number_input(input_values, predicate["left"]) != _number_input( + input_values, + predicate["right"], + ) + + if kind == "number-input-not-null": + return input_values[predicate["input"]] is not None + + if kind == "supported-protocol": + result = input_values[predicate["input"]] in TUS_SUPPORTED_PROTOCOLS + return result == predicate["equals"] + + raise ValueError("Unsupported generated start validation predicate {}".format(kind)) + + +def _client_flow_value(name): + return START_VALIDATION_CLIENT_FLOW_VALUES[name] + + +def _number_input(input_values, name): + value = input_values[name] + if value is None: + return 0 + + return value + + +def _resolve_message(message, input_values): + template = START_VALIDATION_MESSAGES[message["key"]] + if message["kind"] == "client-flow-message": + return template + + if message["kind"] == "client-flow-message-with-input-suffix": + return "{}{}".format(template, input_values[message["input"]]) + + raise ValueError( + "Unsupported generated start validation message {}".format(message["kind"]) + ) diff --git a/tusclient/storage/filestorage.py b/tusclient/storage/filestorage.py index 05a6fb5..b7e50a7 100644 --- a/tusclient/storage/filestorage.py +++ b/tusclient/storage/filestorage.py @@ -1,8 +1,16 @@ """ An implementation of , using a file as storage. """ +from random import random + from tinydb import TinyDB, Query +from tusclient.protocol_generated import ( + url_storage_fingerprint_prefix, + url_storage_id, + url_storage_key, +) + from . import interface @@ -19,7 +27,7 @@ def get_item(self, key: str): - key[str]: The unique id for the stored item (in this case, url) :Returns: url[str] """ - result = self._db.search(self._urls.key == key) + result = self._records_for_item(key) return result[0].get("url") if result else None def set_item(self, key: str, url: str): @@ -30,19 +38,45 @@ def set_item(self, key: str, url: str): - key[str]: The unique id to which the item (in this case, url) would be stored. - value[str]: The actual url value to be stored. """ - if self._db.search(self._urls.key == key): - self._db.update({"url": url}, self._urls.key == key) + result = self._records_for_item(key) + if result: + self._db.update({"url": url}, self._urls.key == result[0].get("key")) else: - self._db.insert({"key": key, "url": url}) + self._db.insert({"key": self._new_storage_key(key), "url": url}) def remove_item(self, key: str): """ Remove/Delete the url value under the unique key from storage. """ - self._db.remove(self._urls.key == key) + for stored_key in self._keys_for_item(key): + self._db.remove(self._urls.key == stored_key) def close(self): """ Close the file storage and release all opened files. """ self._db.close() + + def count(self): + return len(self._db.all()) + + def keys(self): + return [record.get("key") for record in self._db.all()] + + def _records_for_item(self, key: str): + exact = self._db.search(self._urls.key == key) + if exact: + return exact + + prefix = url_storage_fingerprint_prefix(key) + return self._db.search( + self._urls.key.test( + lambda stored_key: isinstance(stored_key, str) and stored_key.startswith(prefix) + ) + ) + + def _keys_for_item(self, key: str): + return [record.get("key") for record in self._records_for_item(key)] + + def _new_storage_key(self, key: str): + return url_storage_key(key, url_storage_id(random())) diff --git a/tusclient/upload_chunks_generated.py b/tusclient/upload_chunks_generated.py new file mode 100644 index 0000000..49605b5 --- /dev/null +++ b/tusclient/upload_chunks_generated.py @@ -0,0 +1,84 @@ +# Code generated from Transloadit API2 TUS protocol contracts; DO NOT EDIT. +# If it looks wrong, please report the issue instead of editing this file by hand; +# the source fix belongs in the protocol contract generator so all TUS clients stay in sync. + +import time + +from tusclient.abort_generated import next_retry_attempt +from tusclient.exceptions import TusCommunicationError, TusUploadFailed + + +RETRY_ATTEMPT_RESET_POLICY = 'when-offset-advanced-since-last-retry' + + +def effective_retry_attempt(retry_attempt, offset, offset_before_retry): + if RETRY_ATTEMPT_RESET_POLICY == 'when-offset-advanced-since-last-retry': + if offset > offset_before_retry: + return 0 + return retry_attempt + raise ValueError( + 'tus: unsupported retry reset policy {}'.format( + RETRY_ATTEMPT_RESET_POLICY, + ) + ) + + +def should_schedule_upload_retry(on_should_retry, error, retry_attempt, retry_delays): + if retry_attempt >= len(retry_delays): + return False + if on_should_retry is not None: + return bool(on_should_retry(error, retry_attempt)) + return True + + +def upload_chunk_with_retry( + uploader, + perform_patch_request, + retry_delays, + on_should_retry=None, +): + uploader._retried = 0 + offset_before_retry = uploader.offset + start_offset = uploader.offset + + if not uploader.upload_length_deferred: + uploader.notify_progress(start_offset) + + while True: + error = None + try: + perform_patch_request() + except TusUploadFailed as patch_error: + error = patch_error + if error is None: + if uploader.upload_length_deferred: + uploader.notify_progress(start_offset) + uploader.notify_progress(uploader.offset) + uploader.notify_chunk_complete(uploader.offset - start_offset, uploader.offset) + return + + while error is not None: + effective_attempt = effective_retry_attempt( + uploader._retried, + uploader.offset, + offset_before_retry, + ) + if not should_schedule_upload_retry( + on_should_retry, + error, + effective_attempt, + retry_delays, + ): + raise error + + delay_ms = retry_delays[effective_attempt] + if delay_ms > 0: + time.sleep(delay_ms / 1000) + uploader._retried = next_retry_attempt(effective_attempt) + offset_before_retry = uploader.offset + error = None + try: + uploader.offset = uploader.get_offset() + except TusCommunicationError as sync_error: + error = sync_error + continue diff --git a/tusclient/uploader/baseuploader.py b/tusclient/uploader/baseuploader.py index e5f0379..4d9a90b 100644 --- a/tusclient/uploader/baseuploader.py +++ b/tusclient/uploader/baseuploader.py @@ -1,15 +1,37 @@ -from typing import Optional, IO, Dict, Tuple, TYPE_CHECKING, Union +from typing import Callable, Optional, IO, Dict, Tuple, TYPE_CHECKING, Union import os import re from base64 import b64encode from sys import maxsize as MAXSIZE import hashlib +from threading import Event import requests from tusclient.exceptions import TusCommunicationError from tusclient.request import TusRequest, catch_requests_error from tusclient.fingerprint import fingerprint, interface +from tusclient.protocol_generated import ( + DEFAULT_REQUEST_HEADERS, + PARALLEL_FINAL_CONCAT_PREFIX, + PARALLEL_PARTIAL_HEADER_KIND, + PARALLEL_PARTIAL_METADATA_SOURCE, + PARALLEL_PARTIAL_NESTED_UPLOADS, + PARALLEL_PARTIAL_URL_STORAGE, + PARALLEL_UPLOAD_DEFAULT, + PARALLEL_UPLOAD_MINIMUM, + PARALLEL_UPLOAD_SPLIT_STRATEGY, + PARALLEL_UPLOAD_URL_SEPARATOR, + START_VALIDATION_MESSAGES, + METADATA_HEADER_NAME, + UPLOAD_CONCAT_HEADER_NAME, + UPLOAD_CONCAT_PARTIAL_VALUE, + UPLOAD_DEFER_LENGTH_HEADER_NAME, + UPLOAD_LENGTH_HEADER_NAME, + prepare_request_headers, +) +from tusclient.request_lifecycle import TusRequestContext +from tusclient.start_validation import validate_upload_start_or_raise from tusclient.storage.interface import Storage if TYPE_CHECKING: @@ -61,6 +83,8 @@ class BaseUploader: If not specified, it defaults to True. - store_url (bool): Determines whether or not url should be stored, and uploads should be resumed. + - remove_fingerprint_on_success (bool): + Determines whether the stored upload URL should be removed after a successful upload. - url_storage (): An implementation of which is an API for URL storage. This value must be set if store_url is set to true. A ready to use implementation exists atbe used out of the box. But you can @@ -77,6 +101,12 @@ class BaseUploader: Whether or not to declare the upload length when finished reading the file stream instead of when the upload is started. This is useful when uploading from a streaming resource, where the total file size isn't available when the upload is created but only becomes known when the stream finishes. The server must support the `creation-defer-length` extension. + - on_progress (Optional[Callable]): + Callback invoked with bytes sent and total bytes after upload progress changes. + - on_chunk_complete (Optional[Callable]): + Callback invoked with chunk size, accepted offset, and total bytes after a chunk is accepted. + - on_should_retry (Optional[Callable]): + Callback invoked with an error and retry attempt before scheduling a retry. :Constructor Args: - file_path (str) @@ -90,13 +120,14 @@ class BaseUploader: - retry_delay (Optional[int]) - verify_tls_cert (Optional[bool]) - store_url (Optional[bool]) + - remove_fingerprint_on_success (Optional[bool]) - url_storage (Optinal []) - fingerprinter (Optional []) - upload_checksum (Optional[bool]) - upload_length_deferred (Optional[bool]) """ - DEFAULT_HEADERS = {"Tus-Resumable": "1.0.0"} + DEFAULT_HEADERS = dict(DEFAULT_REQUEST_HEADERS) DEFAULT_CHUNK_SIZE = MAXSIZE CHECKSUM_ALGORITHM_PAIR = ( "sha1", @@ -116,10 +147,22 @@ def __init__( retry_delay: int = 30, verify_tls_cert: bool = True, store_url=False, + remove_fingerprint_on_success=False, url_storage: Optional[Storage] = None, fingerprinter: Optional[interface.Fingerprint] = None, upload_checksum=False, upload_length_deferred=False, + upload_size: Optional[int] = None, + upload_data_during_creation=False, + override_patch_method=False, + parallel_uploads: Optional[int] = None, + parallel_upload_boundaries=None, + metadata_for_partial_uploads: Optional[Dict] = None, + protocol: Optional[str] = None, + retry_delays=None, + on_should_retry: Optional[Callable[[Exception, int], bool]] = None, + on_progress: Optional[Callable[[int, Optional[int]], None]] = None, + on_chunk_complete: Optional[Callable[[int, int, Optional[int]], None]] = None, ): if file_path is None and file_stream is None: raise ValueError("Either 'file_path' or 'file_stream' cannot be None.") @@ -132,6 +175,20 @@ def __init__( "Please specify a storage instance to enable resumablility." ) + validate_upload_start_or_raise( + client=client, + file_path=file_path, + file_stream=file_stream, + parallel_upload_boundaries=parallel_upload_boundaries, + parallel_uploads=parallel_uploads, + protocol=protocol, + retry_delays=retry_delays, + upload_data_during_creation=upload_data_during_creation, + upload_length_deferred=upload_length_deferred, + upload_size=upload_size, + url=url, + ) + self.verify_tls_cert = verify_tls_cert self.file_path = file_path self.file_stream = file_stream @@ -141,8 +198,10 @@ def __init__( self.metadata = metadata or {} self.metadata_encoding = metadata_encoding self.store_url = store_url + self.remove_fingerprint_on_success = remove_fingerprint_on_success self.url_storage = url_storage self.fingerprinter = fingerprinter or fingerprint.Fingerprint() + self.protocol = protocol self.offset = 0 self.url = None self.__init_url_and_offset(url) @@ -152,29 +211,109 @@ def __init__( self._retried = 0 self.retry_delay = retry_delay self.upload_checksum = upload_checksum + self.upload_data_during_creation = upload_data_during_creation self.upload_length_deferred = upload_length_deferred + self.upload_size = upload_size + self.parallel_uploads = parallel_uploads + self.parallel_upload_boundaries = parallel_upload_boundaries + self.metadata_for_partial_uploads = metadata_for_partial_uploads or {} + self.override_patch_method = override_patch_method + self.retry_delays = retry_delays + self.on_should_retry = on_should_retry + self.on_progress = on_progress + self.on_chunk_complete = on_chunk_complete + self._abort_requested = Event() ( self.__checksum_algorithm_name, self.__checksum_algorithm, ) = self.CHECKSUM_ALGORITHM_PAIR + def abort(self): + self._abort_requested.set() + + def is_aborted(self): + return self._abort_requested.is_set() + def get_headers(self): """ Return headers of the uploader instance. This would include the headers of the client instance. """ + return self.prepare_request_headers() + + def prepare_request_headers(self, operation_headers=None): client_headers = getattr(self.client, "headers", {}) - return dict(self.DEFAULT_HEADERS, **client_headers) + add_request_id = getattr(self.client, "add_request_id", False) + return prepare_request_headers( + operation_headers, + client_headers, + add_request_id, + self.protocol, + ) + + def request_method_input_options(self): + return { + "override_patch_method": self.override_patch_method, + } + + def _upload_retry_delays_ms(self): + """The retry budget as the delays-indexed-by-attempt list (in milliseconds). - def get_url_creation_headers(self): + The legacy ``retries``/``retry_delay`` options map onto it as ``retries`` + repetitions of the same delay, which preserves their historical budget and + sleep behavior through the generated retry runtime. + """ + if self.retry_delays is not None: + return list(self.retry_delays) + + return [self.retry_delay * 1000] * self.retries + + def run_before_request(self, method, url, headers): + if self.client is not None: + self.client._set_current_uploader(self) + context = TusRequestContext(method, url, headers) + hooks = getattr(self.client, "request_hooks", None) + if hooks is not None and hooks.before_request is not None: + hooks.before_request(context) + return context + + def clear_current_request(self): + if self.client is not None: + self.client._clear_current_uploader(self) + + def run_after_response(self, context, response): + hooks = getattr(self.client, "request_hooks", None) + if hooks is not None and hooks.after_response is not None: + hooks.after_response(context, response) + + def get_url_creation_headers( + self, + metadata=None, + partial=False, + upload_length=None, + final_upload_urls=None, + ): """Return headers required to create upload url""" - headers = self.get_headers() - if self.upload_length_deferred: - headers['upload-defer-length'] = '1' + operation_headers = {} + if final_upload_urls is not None: + operation_headers[UPLOAD_CONCAT_HEADER_NAME] = "{}{}".format( + PARALLEL_FINAL_CONCAT_PREFIX, + PARALLEL_UPLOAD_URL_SEPARATOR.join(final_upload_urls), + ) else: - headers["upload-length"] = str(self.file_size) - headers["upload-metadata"] = ",".join(self.encode_metadata()) - return headers + if partial: + operation_headers[UPLOAD_CONCAT_HEADER_NAME] = UPLOAD_CONCAT_PARTIAL_VALUE + if self.upload_length_deferred: + operation_headers[UPLOAD_DEFER_LENGTH_HEADER_NAME] = "1" + else: + operation_headers[UPLOAD_LENGTH_HEADER_NAME] = str( + self.file_size if upload_length is None else upload_length + ) + + encoded_metadata = self.encode_metadata(metadata) + if encoded_metadata: + operation_headers[METADATA_HEADER_NAME] = ",".join(encoded_metadata) + return self.prepare_request_headers(operation_headers) @property def checksum_algorithm(self): @@ -201,9 +340,15 @@ def get_offset(self): This is different from the instance attribute 'offset' because this makes an http request to the tus server to retrieve the offset. """ - resp = requests.head( - self.url, headers=self.get_headers(), verify=self.verify_tls_cert, cert=self.client_cert - ) + headers = self.get_headers() + context = self.run_before_request("HEAD", self.url, headers) + try: + resp = requests.head( + self.url, headers=context.headers, verify=self.verify_tls_cert, cert=self.client_cert + ) + finally: + self.clear_current_request() + self.run_after_response(context, resp) offset = resp.headers.get("upload-offset") if offset is None: msg = "Attempt to retrieve offset fails with status {}".format( @@ -212,12 +357,13 @@ def get_offset(self): raise TusCommunicationError(msg, resp.status_code, resp.content) return int(offset) - def encode_metadata(self): + def encode_metadata(self, metadata=None): """ Return list of encoded metadata as defined by the Tus protocol. """ encoded_list = [] - for key, value in self.metadata.items(): + metadata = self.metadata if metadata is None else metadata + for key, value in metadata.items(): key_str = str(key) # dict keys may be of any object type. # confirm that the key does not contain unwanted characters. @@ -231,6 +377,106 @@ def encode_metadata(self): ) return encoded_list + def parallel_upload_count(self): + parallel_uploads = ( + PARALLEL_UPLOAD_DEFAULT + if self.parallel_uploads is None + else self.parallel_uploads + ) + if parallel_uploads == 1: + return parallel_uploads + if parallel_uploads < PARALLEL_UPLOAD_MINIMUM: + raise ValueError( + "tus: parallel uploads must be at least {}".format( + PARALLEL_UPLOAD_MINIMUM, + ) + ) + + return parallel_uploads + + def parallel_upload_part_ranges(self, parallel_uploads): + if self.parallel_upload_boundaries is not None: + return [ + self._parallel_upload_boundary_range(index, boundary) + for index, boundary in enumerate(self.parallel_upload_boundaries) + ] + + if PARALLEL_UPLOAD_SPLIT_STRATEGY != "contiguous-floor-size-last-remainder": + raise ValueError( + "tus: unsupported parallel upload split strategy {}".format( + PARALLEL_UPLOAD_SPLIT_STRATEGY, + ) + ) + if self.file_size is None: + raise ValueError(START_VALIDATION_MESSAGES["parallelUploadMissingSize"]) + if parallel_uploads <= 0: + raise ValueError("tus: parallel upload count must be positive") + + part_size = self.file_size // parallel_uploads + if part_size <= 0: + raise ValueError("tus: parallel upload parts must not be empty") + + ranges = [] + start = 0 + for index in range(parallel_uploads): + end = start + part_size + if index == parallel_uploads - 1: + end = self.file_size + ranges.append((start, end)) + start = end + + return ranges + + def assert_parallel_upload_policy_supported(self): + if PARALLEL_PARTIAL_HEADER_KIND != "partial-upload": + raise ValueError( + "tus: unsupported partial upload header kind {}".format( + PARALLEL_PARTIAL_HEADER_KIND, + ) + ) + if PARALLEL_PARTIAL_METADATA_SOURCE != "metadataForPartialUploads": + raise ValueError( + "tus: unsupported parallel partial metadata policy {}".format( + PARALLEL_PARTIAL_METADATA_SOURCE, + ) + ) + if PARALLEL_PARTIAL_NESTED_UPLOADS != "disabled": + raise ValueError( + "tus: unsupported nested parallel upload policy {}".format( + PARALLEL_PARTIAL_NESTED_UPLOADS, + ) + ) + if PARALLEL_PARTIAL_URL_STORAGE != "parent-managed": + raise ValueError( + "tus: unsupported parallel partial URL storage policy {}".format( + PARALLEL_PARTIAL_URL_STORAGE, + ) + ) + + def read_file_range(self, start, end): + stream = self.get_file_stream() + try: + stream.seek(start) + chunk = stream.read(end - start) + finally: + if self.file_stream is None: + stream.close() + + if len(chunk) != end - start: + raise ValueError(START_VALIDATION_MESSAGES["parallelUploadSliceMissingValue"]) + + return chunk + + def _parallel_upload_boundary_range(self, index, boundary): + if isinstance(boundary, dict): + return boundary["start"], boundary["end"] + if isinstance(boundary, (list, tuple)) and len(boundary) == 2: + return boundary[0], boundary[1] + + raise ValueError( + "tus: invalid parallel upload boundary at index {}".format(index) + ) + def __init_url_and_offset(self, url: Optional[str] = None): """ Return the tus upload url. @@ -264,8 +510,12 @@ def __init_url_and_offset(self, url: Optional[str] = None): raise error def _get_fingerprint(self): - with self.get_file_stream() as stream: + stream = self.get_file_stream() + try: return self.fingerprinter.get_fingerprint(stream) + finally: + if self.file_stream is None: + stream.close() def set_url(self, url: str): """Set the upload URL""" @@ -282,6 +532,31 @@ def get_request_length(self): return self.chunk_size return min(self.chunk_size, self.stop_at - self.offset) + def notify_progress(self, bytes_sent: int): + if self.on_progress: + self.on_progress(bytes_sent, self.file_size) + + def notify_chunk_complete(self, chunk_size: int, bytes_accepted: int): + if self.on_chunk_complete: + self.on_chunk_complete(chunk_size, bytes_accepted, self.file_size) + + def remove_url_on_success(self): + if not ( + self.store_url + and self.url_storage + and self.remove_fingerprint_on_success + ): + return + + if self.file_size is None or self.offset < self.file_size: + return + + self.url_storage.remove_item(self._get_fingerprint()) + + def remove_stored_url(self): + if self.store_url and self.url_storage: + self.url_storage.remove_item(self._get_fingerprint()) + def get_file_stream(self): """ Return a file stream instance of the upload. diff --git a/tusclient/uploader/uploader.py b/tusclient/uploader/uploader.py index 9b99bf3..9e269c0 100644 --- a/tusclient/uploader/uploader.py +++ b/tusclient/uploader/uploader.py @@ -1,5 +1,4 @@ from typing import Optional -import time import asyncio from urllib.parse import urljoin @@ -9,8 +8,26 @@ from tusclient.uploader.baseuploader import BaseUploader -from tusclient.exceptions import TusUploadFailed, TusCommunicationError +from tusclient.async_upload_chunks_generated import ( + upload_chunk_with_retry as async_upload_chunk_with_retry, +) +from tusclient.detailed_error import ( + create_upload_request_error, + create_upload_response_error, +) +from tusclient.exceptions import TusUploadAborted, TusUploadFailed, TusCommunicationError +from tusclient.protocol_generated import ( + CREATE_UPLOAD_METHOD, + LOCATION_HEADER_NAME, + UPLOAD_CHUNK_METHOD, + UPLOAD_CHUNK_OPERATION_ID, + UPLOAD_OFFSET_HEADER_NAME, + is_successful_response_status, + request_method_plan, + upload_body_headers, +) from tusclient.request import TusRequest, AsyncTusRequest, catch_requests_error +from tusclient.upload_chunks_generated import upload_chunk_with_retry def _verify_upload(request: TusRequest): @@ -21,6 +38,95 @@ def _verify_upload(request: TusRequest): class Uploader(BaseUploader): + @catch_requests_error + def create_url_with_upload(self, bytes_to_upload: int): + """ + Create a new upload URL and send the first bytes in the creation request. + """ + if bytes_to_upload < 0: + raise ValueError("bytes_to_upload must be non-negative") + if self.upload_length_deferred: + raise ValueError( + "create_url_with_upload cannot be used with upload_length_deferred" + ) + if bytes_to_upload > self.get_file_size(): + raise ValueError("bytes_to_upload cannot exceed the upload size") + + stream = self.get_file_stream() + try: + chunk = stream.read(bytes_to_upload) + finally: + if self.file_stream is None: + stream.close() + + if len(chunk) != bytes_to_upload: + raise ValueError( + "Could only read {} of {} requested upload bytes".format( + len(chunk), + bytes_to_upload, + ) + ) + + headers = self.get_url_creation_headers() + headers.update( + upload_body_headers( + self.protocol, + done=bytes_to_upload == self.get_file_size(), + ) + ) + context = self.run_before_request(CREATE_UPLOAD_METHOD, self.client.url, headers) + try: + resp = requests.request( + CREATE_UPLOAD_METHOD, + self.client.url, + data=chunk, + headers=context.headers, + verify=self.verify_tls_cert, + cert=self.client_cert, + ) + except requests.exceptions.RequestException as error: + if self.is_aborted(): + raise TusUploadAborted() + raise create_upload_request_error(context, error) + finally: + self.clear_current_request() + self.run_after_response(context, resp) + + if not is_successful_response_status(resp.status_code): + raise create_upload_response_error(context, resp) + + url = resp.headers.get(LOCATION_HEADER_NAME) + if url is None: + raise create_upload_response_error(context, resp) + + offset = resp.headers.get(UPLOAD_OFFSET_HEADER_NAME) + if offset is None: + raise create_upload_response_error(context, resp) + + try: + accepted_offset = int(offset) + except ValueError: + raise TusCommunicationError( + "Unexpected accepted upload offset {}".format(offset), + resp.status_code, + resp.content, + ) + if accepted_offset < 0 or accepted_offset > bytes_to_upload: + raise TusCommunicationError( + "Unexpected accepted upload offset {}".format(accepted_offset), + resp.status_code, + resp.content, + ) + + previous_offset = self.offset + self.set_url(urljoin(self.client.url, url)) + self.offset = accepted_offset + self.notify_progress(previous_offset) + self.notify_progress(self.offset) + self.notify_chunk_complete(self.offset - previous_offset, self.offset) + self.remove_url_on_success() + return self.url + def upload(self, stop_at: Optional[int] = None): """ Perform file upload. @@ -34,6 +140,13 @@ def upload(self, stop_at: Optional[int] = None): defaults to the file size. """ self.stop_at = stop_at or self.file_size + parallel_uploads = self.parallel_upload_count() + + if parallel_uploads > 1: + if stop_at is not None and stop_at != self.file_size: + raise ValueError("tus: stop_at is not supported with parallel uploads") + self.upload_parallel(parallel_uploads) + return if not self.url: # Ensure the POST request is performed even for empty files. @@ -45,22 +158,152 @@ def upload(self, stop_at: Optional[int] = None): while self.stop_at is None or (self.offset < self.stop_at): self.upload_chunk() + def upload_parallel(self, parallel_uploads: int): + self.assert_parallel_upload_policy_supported() + part_ranges = self.parallel_upload_part_ranges(parallel_uploads) + partial_urls = [ + self.create_partial_url(end - start) + for start, end in part_ranges + ] + accepted_bytes = 0 + + for part_url, (start, end) in zip(partial_urls, part_ranges): + part_size = end - start + accepted_bytes += self.upload_partial_chunk(part_url, start, end) + self.offset = accepted_bytes + self.notify_progress(self.offset) + self.notify_chunk_complete(part_size, self.offset) + + self.set_url(self.create_final_url(partial_urls)) + self.offset = self.file_size + self.remove_url_on_success() + + @catch_requests_error + def create_partial_url(self, part_size: int): + headers = self.get_url_creation_headers( + metadata=self.metadata_for_partial_uploads, + partial=True, + upload_length=part_size, + ) + context = self.run_before_request(CREATE_UPLOAD_METHOD, self.client.url, headers) + try: + resp = requests.request( + CREATE_UPLOAD_METHOD, + self.client.url, + headers=context.headers, + verify=self.verify_tls_cert, + cert=self.client_cert, + ) + except requests.exceptions.RequestException as error: + if self.is_aborted(): + raise TusUploadAborted() + raise create_upload_request_error(context, error) + finally: + self.clear_current_request() + self.run_after_response(context, resp) + url = resp.headers.get(LOCATION_HEADER_NAME) + if not is_successful_response_status(resp.status_code) or url is None: + raise create_upload_response_error(context, resp) + return urljoin(self.client.url, url) + + @catch_requests_error + def upload_partial_chunk(self, partial_url: str, start: int, end: int): + chunk = self.read_file_range(start, end) + operation_headers = { + UPLOAD_OFFSET_HEADER_NAME: "0", + } + operation_headers.update(upload_body_headers(self.protocol, done=True)) + method_plan = request_method_plan( + UPLOAD_CHUNK_OPERATION_ID, + UPLOAD_CHUNK_METHOD, + self.request_method_input_options(), + ) + operation_headers.update(method_plan["headers"]) + headers = self.prepare_request_headers(operation_headers) + context = self.run_before_request(method_plan["method"], partial_url, headers) + try: + resp = requests.request( + method_plan["method"], + partial_url, + data=chunk, + headers=context.headers, + verify=self.verify_tls_cert, + stream=True, + cert=self.client_cert, + ) + except requests.exceptions.RequestException as error: + if self.is_aborted(): + raise TusUploadAborted() + raise create_upload_request_error(context, error) + finally: + self.clear_current_request() + self.run_after_response(context, resp) + + if not is_successful_response_status(resp.status_code): + raise create_upload_response_error(context, resp) + + accepted_offset = resp.headers.get(UPLOAD_OFFSET_HEADER_NAME) + if accepted_offset is None: + raise create_upload_response_error(context, resp) + + try: + accepted_offset = int(accepted_offset) + except ValueError: + raise TusCommunicationError( + "Unexpected accepted upload offset {}".format(accepted_offset), + resp.status_code, + resp.content, + ) + if accepted_offset != len(chunk): + raise TusCommunicationError( + "Unexpected accepted upload offset {}".format(accepted_offset), + resp.status_code, + resp.content, + ) + + return accepted_offset + + @catch_requests_error + def create_final_url(self, partial_urls): + headers = self.get_url_creation_headers(final_upload_urls=partial_urls) + context = self.run_before_request(CREATE_UPLOAD_METHOD, self.client.url, headers) + try: + resp = requests.request( + CREATE_UPLOAD_METHOD, + self.client.url, + headers=context.headers, + verify=self.verify_tls_cert, + cert=self.client_cert, + ) + except requests.exceptions.RequestException as error: + if self.is_aborted(): + raise TusUploadAborted() + raise create_upload_request_error(context, error) + finally: + self.clear_current_request() + self.run_after_response(context, resp) + url = resp.headers.get(LOCATION_HEADER_NAME) + if not is_successful_response_status(resp.status_code) or url is None: + raise create_upload_response_error(context, resp) + return urljoin(self.client.url, url) + def upload_chunk(self): """ Upload chunk of file. """ - self._retried = 0 - # Ensure that we have a URL, as this is behavior we allowed previously. # See https://github.com/tus/tus-py-client/issues/82. if not self.url: self.set_url(self.create_url()) self.offset = 0 - self._do_request() - self.offset = int(self.request.response_headers.get("upload-offset")) - if self.upload_length_deferred and self.request.stream_eof: - self.stop_at = self.offset + upload_chunk_with_retry( + self, + self._perform_patch_request, + self._upload_retry_delays_ms(), + on_should_retry=self.on_should_retry, + ) + self.remove_url_on_success() @catch_requests_error def create_url(self): @@ -69,41 +312,40 @@ def create_url(self): Makes request to tus server to create a new upload url for the required file upload. """ - resp = requests.post( - self.client.url, - headers=self.get_url_creation_headers(), - verify=self.verify_tls_cert, - cert=self.client_cert, - ) - url = resp.headers.get("location") - if url is None: - msg = "Attempt to retrieve create file url with status {}".format( - resp.status_code + headers = self.get_url_creation_headers() + context = self.run_before_request("POST", self.client.url, headers) + try: + resp = requests.post( + self.client.url, + headers=context.headers, + verify=self.verify_tls_cert, + cert=self.client_cert, ) - raise TusCommunicationError(msg, resp.status_code, resp.content) + except requests.exceptions.RequestException as error: + if self.is_aborted(): + raise TusUploadAborted() + raise create_upload_request_error(context, error) + finally: + self.clear_current_request() + self.run_after_response(context, resp) + url = resp.headers.get("location") + if not is_successful_response_status(resp.status_code) or url is None: + raise create_upload_response_error(context, resp) return urljoin(self.client.url, url) - def _do_request(self): + def _perform_patch_request(self): + """Send one chunk PATCH and absorb its accepted state on success. + + The retry algorithm around this transport step lives in the generated + ``upload_chunk_with_retry`` (see tusclient/upload_chunks_generated.py). + """ self.request = TusRequest(self) - try: - self.request.perform() - _verify_upload(self.request) - except TusUploadFailed as error: - self._retry_or_cry(error) - - def _retry_or_cry(self, error): - if self.retries > self._retried: - time.sleep(self.retry_delay) - - self._retried += 1 - try: - self.offset = self.get_offset() - except TusCommunicationError as err: - self._retry_or_cry(err) - else: - self._do_request() - else: - raise error + self.request.perform() + _verify_upload(self.request) + self.offset = int(self.request.response_headers.get("upload-offset")) + if self.upload_length_deferred and self.request.stream_eof: + self.file_size = self.offset + self.stop_at = self.offset class AsyncUploader(BaseUploader): @@ -123,6 +365,19 @@ async def upload(self, stop_at: Optional[int] = None): defaults to the file size. """ self.stop_at = stop_at or self.file_size + parallel_uploads = self.parallel_upload_count() + + if parallel_uploads > 1: + if stop_at is not None and stop_at != self.file_size: + raise ValueError("tus: stop_at is not supported with parallel uploads") + loop = asyncio.get_running_loop() + await loop.run_in_executor( + None, + Uploader.upload_parallel, + self, + parallel_uploads, + ) + return if not self.url: self.set_url(await self.create_url()) @@ -131,22 +386,32 @@ async def upload(self, stop_at: Optional[int] = None): while self.stop_at is None or (self.offset < self.stop_at): await self.upload_chunk() + def create_partial_url(self, part_size: int): + return Uploader.create_partial_url(self, part_size) + + def upload_partial_chunk(self, partial_url: str, start: int, end: int): + return Uploader.upload_partial_chunk(self, partial_url, start, end) + + def create_final_url(self, partial_urls): + return Uploader.create_final_url(self, partial_urls) + async def upload_chunk(self): """ Upload chunk of file. """ - self._retried = 0 - # Ensure that we have a URL, as this is behavior we allowed previously. # See https://github.com/tus/tus-py-client/issues/82. if not self.url: self.set_url(await self.create_url()) self.offset = 0 - await self._do_request() - self.offset = int(self.request.response_headers.get("upload-offset")) - if self.upload_length_deferred and self.request.stream_eof: - self.stop_at = self.offset + await async_upload_chunk_with_retry( + self, + self._perform_patch_request, + self._upload_retry_delays_ms(), + on_should_retry=self.on_should_retry, + ) + self.remove_url_on_success() async def create_url(self): """ @@ -164,42 +429,41 @@ async def create_url(self): conn = aiohttp.TCPConnector(ssl=ssl_ctx) async with aiohttp.ClientSession(connector=conn) as session: headers = self.get_url_creation_headers() - verify_tls_cert = None if self.verify_tls_cert else False - async with session.post( - self.client.url, headers=headers, ssl=verify_tls_cert - ) as resp: - url = resp.headers.get("location") - if url is None: - msg = ( - "Attempt to retrieve create file url with status {}".format( - resp.status + context = self.run_before_request("POST", self.client.url, headers) + try: + verify_tls_cert = None if self.verify_tls_cert else False + async with session.post( + self.client.url, headers=context.headers, ssl=verify_tls_cert + ) as resp: + self.run_after_response(context, resp) + url = resp.headers.get("location") + if url is None: + msg = ( + "Attempt to retrieve create file url with status {}".format( + resp.status + ) + ) + raise TusCommunicationError( + msg, resp.status, await resp.content.read() ) - ) - raise TusCommunicationError( - msg, resp.status, await resp.content.read() - ) - return urljoin(self.client.url, url) + return urljoin(self.client.url, url) + finally: + self.clear_current_request() except aiohttp.ClientError as error: + if self.is_aborted(): + raise TusUploadAborted() raise TusCommunicationError(error) - async def _do_request(self): + async def _perform_patch_request(self): + """Send one chunk PATCH and absorb its accepted state on success. + + The retry algorithm around this transport step lives in the generated + ``upload_chunk_with_retry`` (see tusclient/async_upload_chunks_generated.py). + """ self.request = AsyncTusRequest(self) - try: - await self.request.perform() - _verify_upload(self.request) - except TusUploadFailed as error: - await self._retry_or_cry(error) - - async def _retry_or_cry(self, error): - if self.retries > self._retried: - await asyncio.sleep(self.retry_delay) - - self._retried += 1 - try: - self.offset = self.get_offset() - except TusCommunicationError as err: - await self._retry_or_cry(err) - else: - await self._do_request() - else: - raise error + await self.request.perform() + _verify_upload(self.request) + self.offset = int(self.request.response_headers.get("upload-offset")) + if self.upload_length_deferred and self.request.stream_eof: + self.file_size = self.offset + self.stop_at = self.offset