Skip to content
This repository was archived by the owner on Sep 4, 2020. It is now read-only.

Commit 84312b3

Browse files
iambmeltiambmelt
authored
Merge pull request #35 from microsoftgraph/chunk_upload
Chunk upload integration from OneDrive
2 parents dc85d19 + 78fc3b5 commit 84312b3

14 files changed

Lines changed: 908 additions & 71 deletions

‎graphsdk/src/androidTest/java/com/microsoft/graph/http/MockConnection.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
public class MockConnection implements IConnection {
1515

1616
private final ITestConnectionData mData;
17-
private HashMap<String, String> mHeaders = new HashMap<String, String>();
17+
private HashMap<String, String> mHeaders = new HashMap<>();
1818
private Boolean mFollowRedirects;
1919

2020
public MockConnection(ITestConnectionData data) {
@@ -28,7 +28,7 @@ public void setFollowRedirects(boolean followRedirects) {
2828

2929
@Override
3030
public void addRequestHeader(String headerName, String headerValue) {
31-
mHeaders.put(headerName,headerValue);
31+
mHeaders.put(headerName, headerValue);
3232
}
3333

3434
@Override
@@ -70,4 +70,10 @@ public String getRequestMethod() {
7070
public int getContentLength() {
7171
return mData.getJsonResponse().length();
7272
}
73+
74+
@Override
75+
public void setContentLength(int length) {
76+
// noop
77+
}
78+
7379
}

‎graphsdk/src/androidTest/java/com/microsoft/graph/http/MockHttpProvider.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,11 @@ public <Result, BodyType> Result send(IHttpRequest request,
102102
return null;
103103
}
104104

105+
@Override
106+
public <Result, BodyType, DeserializeType> Result send(IHttpRequest request, Class<Result> resultClass, BodyType serializable, IStatefulResponseHandler<Result, DeserializeType> handler) throws ClientException {
107+
return null;
108+
}
109+
105110
void setConnectionFactory(final IConnectionFactory factory) {
106111
mConnectionFactory = factory;
107112
}

‎graphsdk/src/main/java/com/microsoft/graph/concurrency/ChunkedUploadProvider.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
// ------------------------------------------------------------------------------
2+
// Copyright (c) 2015 Microsoft Corporation
3+
//
4+
// Permission is hereby granted, free of charge, to any person obtaining a copy
5+
// of this software and associated documentation files (the "Software"), to deal
6+
// in the Software without restriction, including without limitation the rights
7+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8+
// copies of the Software, and to permit persons to whom the Software is
9+
// furnished to do so, subject to the following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included in
12+
// all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20+
// THE SOFTWARE.
21+
// ------------------------------------------------------------------------------
22+
23+
package com.microsoft.graph.concurrency;
24+
25+
26+
import com.microsoft.graph.concurrency.ChunkedUploadResponseHandler;
27+
import com.microsoft.graph.concurrency.IProgressCallback;
28+
import com.microsoft.graph.extensions.ChunkedUploadRequest;
29+
import com.microsoft.graph.extensions.ChunkedUploadResult;
30+
import com.microsoft.graph.extensions.IGraphServiceClient;
31+
import com.microsoft.graph.extensions.UploadSession;
32+
import com.microsoft.graph.options.Option;
33+
34+
import java.io.IOException;
35+
import java.io.InputStream;
36+
import java.security.InvalidParameterException;
37+
import java.util.List;
38+
39+
/**
40+
* ChunkedUpload service provider.
41+
*
42+
* @param <UploadType> The upload item type.
43+
*/
44+
public class ChunkedUploadProvider<UploadType> {
45+
46+
/**
47+
* The default chunk size for upload. Currently set to 5 MiB.
48+
*/
49+
private static final int DEFAULT_CHUNK_SIZE = 5 * 1024 * 1024;
50+
51+
/**
52+
* The required chunk size increment by OneDrive Service, which is 320 KiB.
53+
*/
54+
private static final int REQUIRED_CHUNK_SIZE_INCREMENT = 320 * 1024;
55+
56+
/**
57+
* The maximum chunk size for a single upload allowed by OneDrive service.
58+
* Currently the value is 60 MiB.
59+
*/
60+
private static final int MAXIMUM_CHUNK_SIZE = 60 * 1024 * 1024;
61+
62+
/**
63+
* The default retry times for a simple chunk upload if failure happened.
64+
*/
65+
private static final int MAXIMUM_RETRY_TIMES = 3;
66+
67+
/**
68+
* The client.
69+
*/
70+
private final IGraphServiceClient mClient;
71+
72+
/**
73+
* The input stream.
74+
*/
75+
private final InputStream mInputStream;
76+
77+
/**
78+
* The upload session url.
79+
*/
80+
private final String mUploadUrl;
81+
82+
/**
83+
* The stream size.
84+
*/
85+
private final int mStreamSize;
86+
87+
/**
88+
* The upload response handler.
89+
*/
90+
private final ChunkedUploadResponseHandler<UploadType> mResponseHandler;
91+
92+
/**
93+
* The counter for how many bytes read from input stream.
94+
*/
95+
private int mReadSoFar;
96+
97+
/**
98+
* Create the ChunkedUploadProvider
99+
*
100+
* @param uploadSession The initial upload session.
101+
* @param client The onedrive client.
102+
* @param inputStream The input stream.
103+
* @param streamSize The stream size.
104+
* @param uploadTypeClass The upload type class.
105+
*/
106+
public ChunkedUploadProvider(final UploadSession uploadSession,
107+
final IGraphServiceClient client,
108+
final InputStream inputStream,
109+
final int streamSize,
110+
final Class<UploadType> uploadTypeClass) {
111+
if (uploadSession == null) {
112+
throw new InvalidParameterException("Upload session is null.");
113+
}
114+
115+
if (client == null) {
116+
throw new InvalidParameterException("OneDrive client is null.");
117+
}
118+
119+
if (inputStream == null) {
120+
throw new InvalidParameterException("Input stream is null.");
121+
}
122+
123+
if (streamSize <= 0) {
124+
throw new InvalidParameterException("Stream size should larger than 0.");
125+
}
126+
127+
this.mClient = client;
128+
this.mReadSoFar = 0;
129+
this.mInputStream = inputStream;
130+
this.mStreamSize = streamSize;
131+
this.mUploadUrl = uploadSession.uploadUrl;
132+
this.mResponseHandler = new ChunkedUploadResponseHandler(uploadTypeClass);
133+
}
134+
135+
/**
136+
* Upload content to remote upload session based on the input stream.
137+
*
138+
* @param options The upload options.
139+
* @param callback The progress callback invoked during uploading.
140+
* @param configs The optional ocnfigs for the upload options, [0] should be the customized chunk
141+
* size and the [1] should be the maxRetry for upload retry.
142+
* @throws IOException The io exception happend during upload.
143+
*/
144+
public void upload(final List<Option> options,
145+
final IProgressCallback<UploadType> callback,
146+
final int... configs)
147+
throws IOException {
148+
int chunkSize = DEFAULT_CHUNK_SIZE;
149+
150+
if (configs.length > 0) {
151+
chunkSize = configs[0];
152+
}
153+
154+
int maxRetry = MAXIMUM_RETRY_TIMES;
155+
156+
if (configs.length > 1) {
157+
maxRetry = configs[1];
158+
}
159+
160+
if (chunkSize % REQUIRED_CHUNK_SIZE_INCREMENT != 0) {
161+
throw new IllegalArgumentException("Chunk size must be a multiple of 320 KiB");
162+
}
163+
164+
if (chunkSize > MAXIMUM_CHUNK_SIZE) {
165+
throw new IllegalArgumentException("Please set chunk size smaller than 60 MiB");
166+
}
167+
168+
byte[] buffer = new byte[chunkSize];
169+
170+
while (this.mReadSoFar < this.mStreamSize) {
171+
int read = this.mInputStream.read(buffer);
172+
173+
if (read == -1) {
174+
break;
175+
}
176+
177+
ChunkedUploadRequest request =
178+
new ChunkedUploadRequest(this.mUploadUrl, this.mClient, options, buffer, read,
179+
maxRetry, this.mReadSoFar, this.mStreamSize);
180+
ChunkedUploadResult result = request.upload(this.mResponseHandler);
181+
182+
if (result.uploadCompleted()) {
183+
callback.progress(this.mStreamSize, this.mStreamSize);
184+
callback.success((UploadType) result.getItem());
185+
break;
186+
} else if (result.chunkCompleted()) {
187+
callback.progress(this.mReadSoFar, this.mStreamSize);
188+
} else if (result.hasError()) {
189+
callback.failure(result.getError());
190+
break;
191+
}
192+
193+
this.mReadSoFar += read;
194+
}
195+
}
196+
}

‎graphsdk/src/main/java/com/microsoft/graph/concurrency/ChunkedUploadResponseHandler.java‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
// ------------------------------------------------------------------------------
2+
// Copyright (c) 2015 Microsoft Corporation
3+
//
4+
// Permission is hereby granted, free of charge, to any person obtaining a copy
5+
// of this software and associated documentation files (the "Software"), to deal
6+
// in the Software without restriction, including without limitation the rights
7+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8+
// copies of the Software, and to permit persons to whom the Software is
9+
// furnished to do so, subject to the following conditions:
10+
//
11+
// The above copyright notice and this permission notice shall be included in
12+
// all copies or substantial portions of the Software.
13+
//
14+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20+
// THE SOFTWARE.
21+
// ------------------------------------------------------------------------------
22+
23+
package com.microsoft.graph.concurrency;
24+
25+
import com.microsoft.graph.extensions.ChunkedUploadResult;
26+
import com.microsoft.graph.extensions.UploadSession;
27+
import com.microsoft.graph.http.DefaultHttpProvider;
28+
import com.microsoft.graph.http.GraphServiceException;
29+
import com.microsoft.graph.http.HttpResponseCode;
30+
import com.microsoft.graph.http.IConnection;
31+
import com.microsoft.graph.http.IHttpRequest;
32+
import com.microsoft.graph.http.IStatefulResponseHandler;
33+
import com.microsoft.graph.logger.ILogger;
34+
import com.microsoft.graph.serializer.ISerializer;
35+
36+
import java.io.BufferedInputStream;
37+
import java.io.InputStream;
38+
39+
/**
40+
* The class handles the stateful response from OneDrive upload session.
41+
*
42+
* @param <UploadType> The expected uploaded item.
43+
*/
44+
public class ChunkedUploadResponseHandler<UploadType>
45+
implements IStatefulResponseHandler<ChunkedUploadResult, UploadType> {
46+
/**
47+
* The expected deserialize type for upload type.
48+
*/
49+
private final Class<UploadType> mDeserializeTypeClass;
50+
51+
/**
52+
* Create a chunked upload response handler.
53+
*
54+
* @param uploadType The expected upload item type.
55+
*/
56+
public ChunkedUploadResponseHandler(final Class<UploadType> uploadType) {
57+
this.mDeserializeTypeClass = uploadType;
58+
}
59+
60+
/**
61+
* Do nothing before get response.
62+
*
63+
* @param connection The connection.
64+
*/
65+
@Override
66+
public void configConnection(final IConnection connection) {
67+
return;
68+
}
69+
70+
/**
71+
* Generate the chunked upload response result.
72+
*
73+
* @param request The http request.
74+
* @param connection The http connection.
75+
* @param serializer The serializer.
76+
* @param logger The system logger.
77+
* @return The chunked upload result which could be upload session/uploaded item or error.
78+
* @throws Exception An exception occurs if the request was unable to complete for any reason.
79+
*/
80+
@Override
81+
public ChunkedUploadResult generateResult(
82+
final IHttpRequest request,
83+
final IConnection connection,
84+
final ISerializer serializer,
85+
final ILogger logger) throws Exception {
86+
InputStream in = null;
87+
88+
try {
89+
if (connection.getResponseCode() == HttpResponseCode.HTTP_ACCEPTED) {
90+
logger.logDebug("Chunk bytes has been accepted by the server.");
91+
in = new BufferedInputStream(connection.getInputStream());
92+
final UploadSession seesion = serializer.deserializeObject(
93+
DefaultHttpProvider.streamToString(in), UploadSession.class);
94+
95+
return new ChunkedUploadResult(seesion);
96+
97+
} else if (connection.getResponseCode() == HttpResponseCode.HTTP_CREATED
98+
|| connection.getResponseCode() == HttpResponseCode.HTTP_OK) {
99+
logger.logDebug("Upload session is completed, uploaded item returned.");
100+
in = new BufferedInputStream(connection.getInputStream());
101+
String rawJson = DefaultHttpProvider.streamToString(in);
102+
UploadType uploadedItem = serializer.deserializeObject(rawJson,
103+
this.mDeserializeTypeClass);
104+
105+
return new ChunkedUploadResult(uploadedItem);
106+
} else if (connection.getResponseCode() >= HttpResponseCode.HTTP_CLIENT_ERROR) {
107+
logger.logDebug("Receiving error during upload, see detail on result error");
108+
109+
return new ChunkedUploadResult(
110+
GraphServiceException.createFromConnection(request, null, serializer,
111+
connection));
112+
}
113+
} finally {
114+
if (in != null) {
115+
in.close();
116+
}
117+
}
118+
119+
return null;
120+
}
121+
}

0 commit comments

Comments
 (0)