From b11b1923087a6718ae602acb317fc2f84f0fc828 Mon Sep 17 00:00:00 2001 From: Mumus Date: Wed, 30 Mar 2016 14:21:53 +0300 Subject: [PATCH] Fix MetaData.FilesAsync() download request infinite loop problem - When request to the given file fails with specific errors like 404 Not Found, or 461 DMCA Takedown the error was not recognized and we still tried to download the file. --- src/DropboxRestAPI/RequestExecuter.cs | 43 ++-- .../Core/IMetadataRequestGenerator.cs | 2 +- .../Core/MetadataRequestGenerator.cs | 202 +++++++++--------- src/DropboxRestAPI/Services/Core/Metadata.cs | 57 +++-- 4 files changed, 162 insertions(+), 142 deletions(-) diff --git a/src/DropboxRestAPI/RequestExecuter.cs b/src/DropboxRestAPI/RequestExecuter.cs index db47769..ba8984b 100644 --- a/src/DropboxRestAPI/RequestExecuter.cs +++ b/src/DropboxRestAPI/RequestExecuter.cs @@ -116,7 +116,7 @@ public RequestExecuter(HttpClient clientContent, HttpClient clientOAuth) } if (statusCode == 0) - throw new HttpException((int) statusCode, content) {Attempts = 1}; + throw new HttpException((int)statusCode, content) { Attempts = 1 }; if ((int)statusCode == 429 || statusCode == HttpStatusCode.ServiceUnavailable) { @@ -126,21 +126,26 @@ public RequestExecuter(HttpClient clientContent, HttpClient clientOAuth) if (retryAfter.Value != null && retryAfter.Value.Any()) { throw new RetryLaterException - { - RetryAfter = decimal.Parse(retryAfter.Value.First(), CultureInfo.InvariantCulture) - }; + { + RetryAfter = decimal.Parse(retryAfter.Value.First(), CultureInfo.InvariantCulture) + }; } } - throw new RetryLaterException {RetryAfter = 10}; + throw new RetryLaterException { RetryAfter = 10 }; } - if ((int) statusCode == 507) + if ((int)statusCode == 507) { throw new NotEnoughQuotaException(); } - if (statusCode == HttpStatusCode.Unauthorized || - statusCode == HttpStatusCode.Forbidden || - statusCode == HttpStatusCode.BadRequest || - statusCode == HttpStatusCode.ServiceUnavailable) + + + if (statusCode == HttpStatusCode.InternalServerError || + statusCode == HttpStatusCode.BadGateway) + { + throw new HttpException((int)statusCode, content ?? httpResponse.ReasonPhrase) { Attempts = int.MaxValue }; + } + + if ((int)statusCode >= 400) { Error errorInfo = null; @@ -161,24 +166,20 @@ public RequestExecuter(HttpClient clientContent, HttpClient clientOAuth) if (!string.IsNullOrEmpty(content)) errorInfo = JsonConvert.DeserializeObject(content); if (errorInfo == null || errorInfo.error == null) - throw new HttpException((int) statusCode, content) {Attempts = 1}; + throw new HttpException((int)statusCode, content ?? httpResponse.ReasonPhrase) { Attempts = 1 }; string error = errorInfo.error; if (!string.IsNullOrEmpty(errorInfo.error_description)) error = string.Format("{0}: {1}", errorInfo.error, errorInfo.error_description); - throw new ServiceErrorException((int) statusCode, error) - { - ErrorCode = errorInfo.error, - ErrorDescription = errorInfo.error_description - }; - } - if (statusCode == HttpStatusCode.InternalServerError || - statusCode == HttpStatusCode.BadGateway) - { - throw new HttpException((int) statusCode, content) {Attempts = int.MaxValue}; + throw new ServiceErrorException((int)statusCode, error) + { + ErrorCode = httpResponse.ReasonPhrase, + ErrorDescription = errorInfo.error_description + }; } + return content; } diff --git a/src/DropboxRestAPI/RequestsGenerators/Core/IMetadataRequestGenerator.cs b/src/DropboxRestAPI/RequestsGenerators/Core/IMetadataRequestGenerator.cs index c969e51..9e0838a 100644 --- a/src/DropboxRestAPI/RequestsGenerators/Core/IMetadataRequestGenerator.cs +++ b/src/DropboxRestAPI/RequestsGenerators/Core/IMetadataRequestGenerator.cs @@ -30,7 +30,7 @@ namespace DropboxRestAPI.RequestsGenerators.Core { public interface IMetadataRequestGenerator { - IRequest Files(string root, string path, string rev = null, string asTeamMember = null); + IRequest Files(string root, string path, string rev = null, string asTeamMember = null, bool useGetHttpMethod = false); IRequest FilesRange(string root, string path, long from, long to, string etag, string rev = null, string asTeamMember = null); IRequest FilesPut(string root, Stream content, string path, string locale = null, bool overwrite = true, string parent_rev = null, bool autorename = true, string asTeamMember = null); diff --git a/src/DropboxRestAPI/RequestsGenerators/Core/MetadataRequestGenerator.cs b/src/DropboxRestAPI/RequestsGenerators/Core/MetadataRequestGenerator.cs index 19696a5..f98c32e 100644 --- a/src/DropboxRestAPI/RequestsGenerators/Core/MetadataRequestGenerator.cs +++ b/src/DropboxRestAPI/RequestsGenerators/Core/MetadataRequestGenerator.cs @@ -32,14 +32,14 @@ namespace DropboxRestAPI.RequestsGenerators.Core { public class MetadataRequestGenerator : IMetadataRequestGenerator { - public IRequest Files(string root, string path, string rev = null, string asTeamMember = null) + public IRequest Files(string root, string path, string rev = null, string asTeamMember = null, bool useGetHttpMethod = false) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/files/" + root + path.EncodePathParts(), - Method = HttpMethod.Head - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/files/" + root + path.EncodePathParts(), + Method = useGetHttpMethod ? HttpMethod.Get : HttpMethod.Head + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); if (!string.IsNullOrEmpty(rev)) @@ -51,11 +51,11 @@ public IRequest Files(string root, string path, string rev = null, string asTeam public IRequest FilesRange(string root, string path, long @from, long to, string etag, string rev = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/files/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/files/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); if (to < 0) @@ -73,11 +73,11 @@ public IRequest FilesRange(string root, string path, long @from, long to, string public IRequest FilesPut(string root, Stream content, string path, string locale = null, bool overwrite = true, string parent_rev = null, bool autorename = true, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/files_put/" + root + path.EncodePathParts(), - Method = HttpMethod.Put - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/files_put/" + root + path.EncodePathParts(), + Method = HttpMethod.Put + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddHeader("locale", locale); @@ -99,11 +99,11 @@ public IRequest Metadata(string root, string path, int file_limit = 10000, strin bool include_membership = false, string asTeamMember = null) { var request = new Request - { - Method = HttpMethod.Get, - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/metadata/" + root + path.EncodePathParts() - }; + { + Method = HttpMethod.Get, + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/metadata/" + root + path.EncodePathParts() + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("hash", hash); @@ -128,11 +128,11 @@ public IRequest Delta(string cursor = null, string locale = null, string path_pr string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/delta", - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/delta", + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("cursor", cursor); @@ -147,11 +147,11 @@ public IRequest Delta(string cursor = null, string locale = null, string path_pr public IRequest DeltaLatestCursor(string path_prefix = null, bool include_media_info = false, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/delta/latest_cursor", - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/delta/latest_cursor", + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("path_prefix", path_prefix); @@ -164,11 +164,11 @@ public IRequest DeltaLatestCursor(string path_prefix = null, bool include_media_ public IRequest LongPollDelta(string cursor = null, int timeout = 30, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiNotifyBaseUrl, - Resource = Consts.Version + "/longpoll_delta", - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiNotifyBaseUrl, + Resource = Consts.Version + "/longpoll_delta", + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("cursor", cursor); @@ -181,11 +181,11 @@ public IRequest LongPollDelta(string cursor = null, int timeout = 30, string asT public IRequest Revisions(string root, string path, int rev_limit = 10, string locale = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/revisions/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/revisions/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("locale", locale); @@ -198,11 +198,11 @@ public IRequest Revisions(string root, string path, int rev_limit = 10, string l public IRequest Restore(string root, string path, string rev = null, string locale = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/restore/" + root + path.EncodePathParts(), - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/restore/" + root + path.EncodePathParts(), + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("rev", rev); @@ -215,11 +215,11 @@ public IRequest Search(string root, string path, string query, int file_limit = string locale = null, bool include_membership = false, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/search/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/search/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("query", query); @@ -237,11 +237,11 @@ public IRequest Search(string root, string path, string query, int file_limit = public IRequest Shares(string root, string path, string locale = null, bool short_url = true, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/shares/" + root + path.EncodePathParts(), - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/shares/" + root + path.EncodePathParts(), + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("locale", locale); @@ -254,11 +254,11 @@ public IRequest Shares(string root, string path, string locale = null, bool shor public IRequest Media(string root, string path, string locale = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/media/" + root + path.EncodePathParts(), - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/media/" + root + path.EncodePathParts(), + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("locale", locale); @@ -269,11 +269,11 @@ public IRequest Media(string root, string path, string locale = null, string asT public IRequest CopyRef(string root, string path, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/copy_ref/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/copy_ref/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); return request; @@ -282,11 +282,11 @@ public IRequest CopyRef(string root, string path, string asTeamMember = null) public IRequest Thumbnails(string root, string path, string format = "jpeg", string size = "s", string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/thumbnails/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/thumbnails/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("format", format); @@ -298,11 +298,11 @@ public IRequest Thumbnails(string root, string path, string format = "jpeg", str public IRequest Previews(string root, string path, string rev = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/previews/" + root + path.EncodePathParts(), - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/previews/" + root + path.EncodePathParts(), + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("rev", rev); @@ -314,11 +314,11 @@ public IRequest Previews(string root, string path, string rev = null, string asT public IRequest ChunkedUpload(byte[] content, int count, string uploadId = null, long? offset = null, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/chunked_upload/", - Method = HttpMethod.Put - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/chunked_upload/", + Method = HttpMethod.Put + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("upload_id", uploadId); @@ -335,11 +335,11 @@ public IRequest CommitChunkedUpload(string root, string path, string uploadId, s string parent_rev = null, bool autorename = true, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiContentBaseUrl, - Resource = Consts.Version + "/commit_chunked_upload/" + root + path.EncodePathParts(), - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiContentBaseUrl, + Resource = Consts.Version + "/commit_chunked_upload/" + root + path.EncodePathParts(), + Method = HttpMethod.Post + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); request.AddParameter("upload_id", uploadId); @@ -359,11 +359,11 @@ public IRequest SharedFolders(string shared_folder_id = null, bool? include_memb throw new ArgumentException("'include_membership' is required if shared_folder_id is specified.", "include_membership"); var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/shared_folders/" + shared_folder_id, - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/shared_folders/" + shared_folder_id, + Method = HttpMethod.Get + }; if (include_membership != null) request.AddParameter("include_membership", include_membership.Value.ToString()); if (show_unmounted) @@ -376,11 +376,11 @@ public IRequest SharedFolders(string shared_folder_id = null, bool? include_memb public IRequest SaveUrl(string root, string path, string url, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/save_url/" + root + path.EncodePathParts(), - Method = HttpMethod.Post - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/save_url/" + root + path.EncodePathParts(), + Method = HttpMethod.Post + }; request.AddParameter("url", url); request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); @@ -390,11 +390,11 @@ public IRequest SaveUrl(string root, string path, string url, string asTeamMembe public IRequest SaveUrlJob(string jobId, string asTeamMember = null) { var request = new Request - { - BaseAddress = Consts.ApiBaseUrl, - Resource = Consts.Version + "/save_url_job/" + jobId, - Method = HttpMethod.Get - }; + { + BaseAddress = Consts.ApiBaseUrl, + Resource = Consts.Version + "/save_url_job/" + jobId, + Method = HttpMethod.Get + }; request.AddHeader(Consts.AsTeamMemberHeader, asTeamMember); return request; diff --git a/src/DropboxRestAPI/Services/Core/Metadata.cs b/src/DropboxRestAPI/Services/Core/Metadata.cs index db5a6eb..8e3db34 100644 --- a/src/DropboxRestAPI/Services/Core/Metadata.cs +++ b/src/DropboxRestAPI/Services/Core/Metadata.cs @@ -33,6 +33,7 @@ using System.Threading.Tasks; using DropboxRestAPI.Models.Core; using DropboxRestAPI.RequestsGenerators.Core; +using DropboxRestAPI.Utils; using Newtonsoft.Json; namespace DropboxRestAPI.Services.Core @@ -58,30 +59,48 @@ public async Task FilesAsync(string path, Stream targetStream, string { MetaData fileMetadata = null; string etag = ""; - long? length; - using (var restResponse = await _requestExecuter.Execute(() => _requestGenerator.Files(_options.Root, path, rev, asTeamMember), cancellationToken: cancellationToken).ConfigureAwait(false)) + long? length = null; + try { - await _requestExecuter.CheckForError(restResponse, false, cancellationToken).ConfigureAwait(false); - - length = restResponse.Content.Headers.ContentLength; - if (length == null) + using (var restResponse = await _requestExecuter.Execute( + () => _requestGenerator.Files(_options.Root, path, rev, asTeamMember), + cancellationToken: cancellationToken).ConfigureAwait(false)) { - IEnumerable metadatas; - if (restResponse.Headers.TryGetValues("x-dropbox-metadata", out metadatas)) + await _requestExecuter.CheckForError(restResponse, false, cancellationToken).ConfigureAwait(false); + length = restResponse.Content.Headers.ContentLength; + if (length == null) { - string metadata = metadatas.FirstOrDefault(); - if (metadata != null) + IEnumerable metadatas; + if (restResponse.Headers.TryGetValues("x-dropbox-metadata", out metadatas)) { - fileMetadata = JsonConvert.DeserializeObject(metadata); - length = fileMetadata.bytes; + string metadata = metadatas.FirstOrDefault(); + if (metadata != null) + { + fileMetadata = JsonConvert.DeserializeObject(metadata); + length = fileMetadata.bytes; + } } } + IEnumerable etags; + if (restResponse.Headers.TryGetValues("etag", out etags)) + etag = etags.FirstOrDefault(); + } + } + catch (HttpException) + { + // Retry the Files request now with GET method in order to retrieve the full error message from the request + using (var restResponseWithContent = await _requestExecuter.Execute( + () => _requestGenerator.Files(_options.Root, path, rev, asTeamMember, true), + cancellationToken: cancellationToken).ConfigureAwait(false)) + { + await + _requestExecuter.CheckForError(restResponseWithContent, false, cancellationToken) + .ConfigureAwait(false); } - IEnumerable etags; - if (restResponse.Headers.TryGetValues("etag", out etags)) - etag = etags.FirstOrDefault(); + throw; } + long read = 0; bool hasMore = true; do @@ -186,10 +205,10 @@ public async Task FilesAsync(string path, Stream targetStream, string var restResponse = await _requestExecuter.Execute(() => _requestGenerator.Previews(Options.AutoRoot, path, rev, asTeamMember), cancellationToken: cancellationToken).ConfigureAwait(false); var preview = new Preview - { - Content = await restResponse.Content.ReadAsStreamAsync().ConfigureAwait(false), - ContentType = restResponse.Content.Headers.ContentType.MediaType - }; + { + Content = await restResponse.Content.ReadAsStreamAsync().ConfigureAwait(false), + ContentType = restResponse.Content.Headers.ContentType.MediaType + }; return preview; }