Skip to content

Commit bcabd31

Browse files
pasinpasin
committed
Merge pull request #1183 from couchbase/feature/issue_1108_push_many_attachments
Fixed #1108 - Too many open files error on push replication
2 parents 0ed3f50 + 38f0f1a commit bcabd31

2 files changed

Lines changed: 156 additions & 112 deletions

File tree

‎src/main/java/com/couchbase/lite/replicator/PusherInternal.java‎

Lines changed: 53 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.couchbase.lite.Status;
1313
import com.couchbase.lite.internal.InterfaceAudience;
1414
import com.couchbase.lite.internal.RevisionInternal;
15+
import com.couchbase.lite.support.BlobContentBody;
1516
import com.couchbase.lite.support.CustomFuture;
1617
import com.couchbase.lite.support.HttpClientFactory;
1718
import com.couchbase.lite.support.RemoteRequest;
@@ -21,14 +22,12 @@
2122
import com.couchbase.lite.util.Log;
2223
import com.couchbase.lite.util.Utils;
2324
import com.couchbase.org.apache.http.entity.mime.MultipartEntity;
24-
import com.couchbase.org.apache.http.entity.mime.content.InputStreamBody;
2525
import com.couchbase.org.apache.http.entity.mime.content.StringBody;
2626

2727
import org.apache.http.HttpResponse;
2828
import org.apache.http.client.HttpResponseException;
2929

3030
import java.io.IOException;
31-
import java.io.InputStream;
3231
import java.net.URL;
3332
import java.nio.charset.Charset;
3433
import java.util.ArrayList;
@@ -516,39 +515,33 @@ public void onCompletion(HttpResponse httpResponse, Object result, Throwable e)
516515
*/
517516
@InterfaceAudience.Private
518517
private boolean uploadMultipartRevision(final RevisionInternal revision) {
519-
520-
// holds inputStream for blob to close after using
521-
final List<InputStream> streamList = new ArrayList<InputStream>();
522-
523518
MultipartEntity multiPart = null;
524-
525519
Map<String, Object> revProps = revision.getProperties();
526-
527520
Map<String, Object> attachments = (Map<String, Object>) revProps.get("_attachments");
528521
for (String attachmentKey : attachments.keySet()) {
529522
Map<String, Object> attachment = (Map<String, Object>) attachments.get(attachmentKey);
530523
if (attachment.containsKey("follows")) {
531-
532524
if (multiPart == null) {
533-
534525
multiPart = new MultipartEntity();
535-
536526
try {
537527
String json = Manager.getObjectMapper().writeValueAsString(revProps);
538528
Charset utf8charset = Charset.forName("UTF-8");
539529
byte[] uncompressed = json.getBytes(utf8charset);
540530
byte[] compressed = null;
541531
byte[] data = uncompressed;
542532
String contentEncoding = null;
543-
if (uncompressed.length > RemoteRequest.MIN_JSON_LENGTH_TO_COMPRESS && canSendCompressedRequests()) {
533+
if (uncompressed.length > RemoteRequest.MIN_JSON_LENGTH_TO_COMPRESS &&
534+
canSendCompressedRequests()) {
544535
compressed = Utils.compressByGzip(uncompressed);
545536
if (compressed.length < uncompressed.length) {
546537
data = compressed;
547538
contentEncoding = "gzip";
548539
}
549540
}
550-
// NOTE: StringBody.contentEncoding default value is null. Setting null value to contentEncoding does not cause any impact.
551-
multiPart.addPart("param1", new StringBody(data, "application/json", utf8charset, contentEncoding));
541+
// NOTE: StringBody.contentEncoding default value is null.
542+
// Setting null value to contentEncoding does not cause any impact.
543+
multiPart.addPart("param1", new StringBody(data, "application/json",
544+
utf8charset, contentEncoding));
552545
} catch (IOException e) {
553546
throw new IllegalArgumentException(e);
554547
}
@@ -557,88 +550,65 @@ private boolean uploadMultipartRevision(final RevisionInternal revision) {
557550
BlobStore blobStore = this.db.getAttachmentStore();
558551
String base64Digest = (String) attachment.get("digest");
559552
BlobKey blobKey = new BlobKey(base64Digest);
560-
InputStream blobStream = blobStore.blobStreamForKey(blobKey);
561-
if (blobStream == null) {
562-
Log.w(Log.TAG_SYNC, "Unable to load the blob stream for blobKey: %s - Skipping upload of multipart revision.", blobKey);
563-
return false;
564-
} else {
565-
streamList.add(blobStream);
566-
String contentType = null;
567-
if (attachment.containsKey("content_type")) {
568-
contentType = (String) attachment.get("content_type");
569-
} else if (attachment.containsKey("type")) {
570-
contentType = (String) attachment.get("type");
571-
} else if (attachment.containsKey("content-type")) {
572-
Log.w(Log.TAG_SYNC, "Found attachment that uses content-type" +
573-
" field name instead of content_type (see couchbase-lite-android" +
574-
" issue #80): %s", attachment);
575-
}
576-
577-
// contentType = null causes Exception from FileBody of apache.
578-
if (contentType == null)
579-
contentType = "application/octet-stream"; // default
580553

581-
// NOTE: Content-Encoding might not be necessary to set. Apache FileBody does not set Content-Encoding.
582-
// FileBody always return null for getContentEncoding(), and Content-Encoding header is not set in multipart
583-
// CBL iOS: https://github.com/couchbase/couchbase-lite-ios/blob/feb7ff5eda1e80bd00e5eb19f1d46c793f7a1951/Source/CBL_Pusher.m#L449-L452
584-
String contentEncoding = null;
585-
if (attachment.containsKey("encoding")) {
586-
contentEncoding = (String) attachment.get("encoding");
587-
}
588-
589-
InputStreamBody inputStreamBody =
590-
new CustomStreamBody(blobStream, contentType,
591-
attachmentKey, contentEncoding);
592-
multiPart.addPart(attachmentKey, inputStreamBody);
554+
String contentType = null;
555+
if (attachment.containsKey("content_type")) {
556+
contentType = (String) attachment.get("content_type");
557+
} else if (attachment.containsKey("type")) {
558+
contentType = (String) attachment.get("type");
559+
} else if (attachment.containsKey("content-type")) {
560+
Log.w(Log.TAG_SYNC, "Found attachment that uses content-type" +
561+
" field name instead of content_type (see couchbase-lite-android" +
562+
" issue #80): %s", attachment);
593563
}
564+
// contentType = null causes Exception from FileBody of apache.
565+
if (contentType == null)
566+
contentType = "application/octet-stream"; // default
567+
568+
// CBL iOS: https://github.com/couchbase/couchbase-lite-ios/blob/feb7ff5eda1e80bd00e5eb19f1d46c793f7a1951/Source/CBL_Pusher.m#L449-L452
569+
String contentEncoding = null;
570+
if (attachment.containsKey("encoding"))
571+
contentEncoding = (String) attachment.get("encoding");
572+
573+
BlobContentBody contentBody = new BlobContentBody(blobStore, blobKey,
574+
contentType, attachmentKey, contentEncoding);
575+
multiPart.addPart(attachmentKey, contentBody);
594576
}
595577
}
596-
597-
if (multiPart == null) {
578+
if (multiPart == null)
598579
return false;
599-
}
600-
601-
final String path = String.format("/%s?new_edits=false", encodeDocumentId(revision.getDocID()));
602580

603581
Log.d(Log.TAG_SYNC, "Uploading multipart request. Revision: %s", revision);
604-
605582
addToChangesCount(1);
606-
607-
CustomFuture future = sendAsyncMultipartRequest("PUT", path, multiPart, new RemoteRequestCompletionBlock() {
608-
@Override
609-
public void onCompletion(HttpResponse httpResponse, Object result, Throwable e) {
610-
try {
611-
if (e != null) {
612-
if (e instanceof HttpResponseException) {
613-
// Server doesn't like multipart, eh? Fall back to JSON.
614-
if (((HttpResponseException) e).getStatusCode() == 415) {
615-
//status 415 = "bad_content_type"
616-
dontSendMultipart = true;
617-
uploadJsonRevision(revision);
618-
}
619-
} else {
620-
Log.e(Log.TAG_SYNC, "Exception uploading multipart request", e);
621-
setError(e);
622-
}
623-
} else {
624-
Log.v(Log.TAG_SYNC, "Uploaded multipart request. Revision: %s", revision);
625-
removePending(revision);
626-
}
627-
} finally {
628-
// close all inputStreams for Blob
629-
for (InputStream stream : streamList) {
583+
final String path = String.format("/%s?new_edits=false", encodeDocumentId(revision.getDocID()));
584+
CustomFuture future = sendAsyncMultipartRequest("PUT", path, multiPart,
585+
new RemoteRequestCompletionBlock() {
586+
@Override
587+
public void onCompletion(HttpResponse httpResponse, Object result, Throwable e) {
630588
try {
631-
stream.close();
632-
} catch (IOException ioe) {
589+
if (e != null) {
590+
if (e instanceof HttpResponseException) {
591+
// Server doesn't like multipart, eh? Fall back to JSON.
592+
if (((HttpResponseException) e).getStatusCode() == 415) {
593+
//status 415 = "bad_content_type"
594+
dontSendMultipart = true;
595+
uploadJsonRevision(revision);
596+
}
597+
} else {
598+
Log.e(Log.TAG_SYNC, "Exception uploading multipart request", e);
599+
setError(e);
600+
}
601+
} else {
602+
Log.v(Log.TAG_SYNC, "Uploaded multipart request. Revision: %s", revision);
603+
removePending(revision);
604+
}
605+
} finally {
606+
addToCompletedChangesCount(1);
633607
}
634608
}
635-
addToCompletedChangesCount(1);
636-
}
637-
}
638-
});
609+
});
639610
future.setQueue(pendingFutures);
640611
pendingFutures.add(future);
641-
642612
return true;
643613
}
644614

@@ -699,35 +669,6 @@ private static int findCommonAncestor(RevisionInternal rev, List<String> possibl
699669
return generation;
700670
}
701671

702-
// CustomFileBody to support contentEncoding. FileBody returns always null for getContentEncoding()
703-
private static class CustomStreamBody extends InputStreamBody {
704-
private String contentEncoding = null;
705-
706-
public CustomStreamBody(final InputStream in, final String mimeType,
707-
final String filename, String contentEncoding) {
708-
super(in, mimeType, filename);
709-
this.contentEncoding = contentEncoding;
710-
}
711-
712-
@Override
713-
protected void finalize() throws Throwable {
714-
// close inputStream after used.
715-
InputStream stream = getInputStream();
716-
if (stream != null) {
717-
try {
718-
stream.close();
719-
} catch (IOException ioe) {
720-
}
721-
}
722-
super.finalize();
723-
}
724-
725-
@Override
726-
public String getContentEncoding() {
727-
return contentEncoding;
728-
}
729-
}
730-
731672
/**
732673
* Submit revisions into inbox for changes from changesSince()
733674
*/

‎src/main/java/com/couchbase/lite/support/BlobContentBody.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/**
2+
* Created by Hideki Itakura on 4/4/16.
3+
* <p/>
4+
* Copyright (c) 2016 Couchbase, Inc All rights reserved.
5+
* <p/>
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
7+
* except in compliance with the License. You may obtain a copy of the License at
8+
* <p/>
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
* <p/>
11+
* Unless required by applicable law or agreed to in writing, software distributed under the
12+
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13+
* either express or implied. See the License for the specific language governing permissions
14+
* and limitations under the License.
15+
*/
16+
package com.couchbase.lite.support;
17+
18+
import com.couchbase.lite.BlobKey;
19+
import com.couchbase.lite.BlobStore;
20+
import com.couchbase.lite.util.Log;
21+
import com.couchbase.org.apache.http.entity.mime.MIME;
22+
import com.couchbase.org.apache.http.entity.mime.content.AbstractContentBody;
23+
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.io.OutputStream;
27+
28+
public class BlobContentBody extends AbstractContentBody {
29+
30+
private final BlobStore blobStore;
31+
private final BlobKey blobKey;
32+
private final String filename;
33+
private final long contentLength;
34+
private final String contentEncoding;
35+
36+
public BlobContentBody(final BlobStore blobStore,
37+
final BlobKey blobKey,
38+
final String mimeType,
39+
final String filename,
40+
final long contentLength,
41+
final String contentEncoding) {
42+
super(mimeType);
43+
if (blobStore == null)
44+
throw new IllegalArgumentException("BlobStore may not be null");
45+
if (blobKey == null)
46+
throw new IllegalArgumentException("BlobKey may not be null");
47+
if (contentLength < -1)
48+
throw new IllegalArgumentException("Content length must be >= -1");
49+
this.blobStore = blobStore;
50+
this.blobKey = blobKey;
51+
this.filename = filename;
52+
this.contentLength = contentLength;
53+
this.contentEncoding = contentEncoding;
54+
}
55+
56+
public BlobContentBody(final BlobStore blobStore,
57+
final BlobKey blobKey,
58+
final String mimeType,
59+
final String filename,
60+
final String contentEncoding) {
61+
this(blobStore, blobKey, mimeType, filename, -1L, contentEncoding);
62+
}
63+
64+
public void writeTo(OutputStream out) throws IOException {
65+
if (out == null)
66+
throw new IllegalArgumentException("Output stream may not be null");
67+
68+
InputStream in = blobStore.blobStreamForKey(blobKey);
69+
if (in == null) {
70+
Log.w(Log.TAG_SYNC, "Unable to load the blob stream for blobKey: " + blobKey);
71+
throw new IOException("Unable to load the blob stream for blobKey: " + blobKey);
72+
}
73+
try {
74+
byte[] tmp = new byte[4096];
75+
int l;
76+
while ((l = in.read(tmp)) != -1)
77+
out.write(tmp, 0, l);
78+
out.flush();
79+
} finally {
80+
in.close();
81+
}
82+
}
83+
84+
public String getTransferEncoding() {
85+
return MIME.ENC_BINARY;
86+
}
87+
88+
public String getCharset() {
89+
return null;
90+
}
91+
92+
public long getContentLength() {
93+
return contentLength;
94+
}
95+
96+
public String getFilename() {
97+
return this.filename;
98+
}
99+
100+
public String getContentEncoding() {
101+
return contentEncoding;
102+
}
103+
}

0 commit comments

Comments
 (0)