Skip to content

Commit 109877c

Browse files
nielshulstaertnielshulstaert
committed
merge of merge request 28 Support for AWS S3 nativelly
1 parent f46aac5 commit 109877c

5 files changed

Lines changed: 54 additions & 61 deletions

File tree

‎MainClass.cs‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ namespace ThermoRawFileParser
88
{
99
public static class MainClass
1010
{
11-
private static readonly log4net.ILog Log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
11+
private static readonly log4net.ILog Log =
12+
log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
1213

1314
public static void Main(string[] args)
1415
{
@@ -25,11 +26,11 @@ public static void Main(string[] args)
2526
string subFolder = null;
2627
string s3url = null;
2728
string s3AccessKeyId = null;
28-
string s3SecretAccessKey = null;
29+
string s3SecretAccessKey = null;
2930
var verbose = false;
3031
string bucketName = null;
31-
var ignoreInstrumentErrors = false;
32-
32+
var ignoreInstrumentErrors = false;
33+
3334
var help = false;
3435

3536
var optionSet = new OptionSet
@@ -79,40 +80,37 @@ public static void Main(string[] args)
7980
},
8081
{
8182
"u:|s3_url:",
82-
"Optional property to write directly the data into S3 Storage",
83+
"Optional property to write directly the data into S3 Storage.",
8384
v => s3url = v
8485
},
8586
{
86-
"k:|s3_accesskeyid:",
87-
"Optional key for the S3 bucket to write the file output",
87+
"k:|s3_accesskeyid:",
88+
"Optional key for the S3 bucket to write the file output.",
8889
v => s3AccessKeyId = v
8990
},
9091
{
91-
"t:|s3_secretaccesskey:",
92-
"Optional key for the S3 bucket to write the file output",
92+
"t:|s3_secretaccesskey:",
93+
"Optional key for the S3 bucket to write the file output.",
9394
v => s3SecretAccessKey = v
9495
},
9596
{
96-
"n:|s3_bucketName:",
97-
"S3 bucket name",
98-
v => bucketName = v
97+
"n:|s3_bucketName:",
98+
"S3 bucket name",
99+
v => bucketName = v
99100
},
100101
{
101-
"v|verbose", "Verbose the programm and the individual steps",
102+
"v|verbose", "Enable verbose logging.",
102103
v => verbose = v != null
103104
},
104105
{
105-
"ignoreInstrumentErrors", "Ignore missing properties by the instrument",
106+
"e|ignoreInstrumentErrors", "Ignore missing properties by the instrument.",
106107
v => ignoreInstrumentErrors = v != null
107108
}
108-
109-
110-
111109
};
112110

113111
try
114112
{
115-
//parse the command line
113+
// parse the command line
116114
var extra = optionSet.Parse(args);
117115

118116
if (!extra.IsNullOrEmpty())
@@ -206,12 +204,15 @@ public static void Main(string[] args)
206204
{
207205
if (verbose)
208206
{
209-
((log4net.Repository.Hierarchy.Hierarchy)LogManager.GetLoggerRepository()).Root.Level = Level.Debug;
210-
((log4net.Repository.Hierarchy.Hierarchy)LogManager.GetLoggerRepository()).RaiseConfigurationChanged(EventArgs.Empty);
211-
207+
((log4net.Repository.Hierarchy.Hierarchy) LogManager.GetRepository()).Root.Level =
208+
Level.Debug;
209+
((log4net.Repository.Hierarchy.Hierarchy) LogManager.GetRepository())
210+
.RaiseConfigurationChanged(EventArgs.Empty);
212211
}
213-
var parseInput = new ParseInput(rawFilePath, outputDirectory, outputFormat, gzip, outputMetadataFormat,
214-
includeProfileData, collection, msRun, subFolder, Log, s3url, s3AccessKeyId, s3SecretAccessKey, bucketName, ignoreInstrumentErrors);
212+
213+
var parseInput = new ParseInput(rawFilePath, outputDirectory, outputFormat, gzip, outputMetadataFormat,
214+
includeProfileData, collection, msRun, subFolder, Log, s3url, s3AccessKeyId, s3SecretAccessKey,
215+
bucketName, ignoreInstrumentErrors);
215216
RawFileParser.Parse(parseInput);
216217
}
217218
catch (Exception ex)

‎Properties/AssemblyInfo.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
// You can specify all the values or you can default the Build and Revision Numbers
3333
// by using the '*' as shown below:
3434
// [assembly: AssemblyVersion("1.0.*")]
35-
[assembly: AssemblyVersion("1.0.6.0")]
36-
[assembly: AssemblyFileVersion("1.0.6.0")]
35+
[assembly: AssemblyVersion("1.0.7.0")]
36+
[assembly: AssemblyFileVersion("1.0.7.0")]
3737

3838
[assembly: log4net.Config.XmlConfigurator(ConfigFile = "log4net.config")]

‎Writer/MzMlSpectrumWriter.cs‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ public class MzMlSpectrumWriter : SpectrumWriter
3535
// Precursor scan number for reference in the precursor element of an MS2 spectrum
3636
private int _precursorScanNumber;
3737

38-
public MzMlSpectrumWriter(ParseInput parseInput , log4net.ILog log ) : base(parseInput)
38+
public MzMlSpectrumWriter(ParseInput parseInput, log4net.ILog log) : base(parseInput)
3939
{
4040
Log = log;
41-
_parseInput = parseInput;
41+
_parseInput = parseInput;
4242
}
4343

4444
/// <inheritdoc />
@@ -97,10 +97,9 @@ public override void Write(IRawDataPlus rawFile, int firstScanNumber, int lastSc
9797
if (_parseInput.S3loader != null)
9898
{
9999
Writer.Flush();
100-
Writer.BaseStream.Position = 0;
101-
_parseInput.S3loader.loadObjectToS3(getFullPath(), getFullPath(), "mzML", getFullPath());
102-
}
103-
100+
Writer.BaseStream.Position = 0;
101+
_parseInput.S3loader.loadObjectToS3(getFullPath(), getFullPath(), "mzML", getFullPath());
102+
}
104103
}
105104
}
106105

@@ -228,7 +227,7 @@ private mzMLType InitializeMzMl()
228227
mzMl.softwareList.software[0] = new SoftwareType
229228
{
230229
id = "ThermoRawFileParser",
231-
version = "1.0.0",
230+
version = "1.0.7",
232231
cvParam = new CVParamType[1]
233232
};
234233

@@ -620,13 +619,13 @@ private SpectrumType ConstructSpectrum(int scanNumber)
620619
}
621620
catch (Exception e)
622621
{
623-
Log.Warn("The IonizationMode do not contains the following property --" + e.Message);
622+
Log.Warn("The IonizationMode does not contains the following property --" + e.Message);
624623
if (!_parseInput.IgnoreInstrumentErrors)
625624
{
626-
throw e;
625+
throw e;
627626
}
628-
629627
}
628+
630629
// Add the mass analyzer if necessary
631630
if (!_massAnalyzers.ContainsKey(scanFilter.MassAnalyzer) &&
632631
OntologyMapping.MassAnalyzerTypes.ContainsKey(scanFilter.MassAnalyzer))
@@ -1206,7 +1205,7 @@ private PrecursorListType ConstructPrecursorList(IScanEventBase scanEvent, int?
12061205
{
12071206
// Do nothing
12081207
}
1209-
}
1208+
}
12101209

12111210
precursor.activation =
12121211
new ParamGroupType

‎Writer/S3Loader.cs‎

Lines changed: 17 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,50 @@
1-

2-
3-
using Amazon.S3;
4-
1+
using Amazon.S3;
52
using Amazon.S3.Model;
63
using System;
74
using System.IO;
85
using Amazon;
96
using Amazon.Runtime;
107

11-
8+
129
namespace ThermoRawFileParser.Writer
1310
{
1411
public class S3Loader
1512
{
1613
private string bucketName;
14+
1715
// Example creates two objects (for simplicity, we upload same file twice).
1816
// You specify key names for these objects.
19-
private static readonly RegionEndpoint bucketRegion = RegionEndpoint.EUWest1;
17+
private static readonly RegionEndpoint bucketRegion = RegionEndpoint.EUWest1;
2018
private static IAmazonS3 client;
2119
private string s3url;
2220
private string s3AccessKeyId;
2321
private string s3SecretAccessKey;
24-
22+
2523

2624
public S3Loader(string s3url, string s3AccessKeyId, string s3SecretAccessKey, string bucketName)
2725
{
28-
2926
this.s3url = s3url;
3027
this.s3AccessKeyId = s3AccessKeyId;
3128
this.s3SecretAccessKey = s3SecretAccessKey;
32-
this.bucketName = bucketName;
29+
this.bucketName = bucketName;
3330
AWSConfigsS3.UseSignatureVersion4 = false;
34-
31+
3532
var s3Config = new AmazonS3Config
3633
{
37-
RegionEndpoint = Amazon.RegionEndpoint.EUWest2,
34+
RegionEndpoint = Amazon.RegionEndpoint.EUWest2,
3835
ForcePathStyle = true,
3936
SignatureVersion = "2",
4037
ServiceURL = s3url,
4138
SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA1
4239
};
43-
40+
4441
client = new AmazonS3Client(new BasicAWSCredentials(s3AccessKeyId, s3SecretAccessKey), s3Config);
4542
this.bucketName = bucketName;
4643

47-
var buckets = client.ListObjects(bucketName);
44+
var buckets = client.ListObjects(bucketName);
4845

49-
if(buckets == null )
46+
if (buckets == null)
5047
throw new AmazonS3Exception("Connection to AWS url -- " + this.s3url);
51-
52-
5348
}
5449

5550
public bool loadObjectToS3(String filePath, string name, string contentType, string label)
@@ -67,21 +62,20 @@ public bool loadObjectToS3(String filePath, string name, string contentType, str
6762
// https://github.com/aws/aws-sdk-net/issues/856. In addition
6863
var s3Config = new AmazonS3Config
6964
{
70-
RegionEndpoint = Amazon.RegionEndpoint.EUWest2,
65+
RegionEndpoint = Amazon.RegionEndpoint.EUWest2,
7166
ForcePathStyle = true,
7267
SignatureVersion = "2",
7368
ServiceURL = s3url,
7469
SignatureMethod = Amazon.Runtime.SigningAlgorithm.HmacSHA1
7570
};
76-
71+
7772
putRequest.Metadata.Add("x-amz-meta-title", label);
7873
putRequest.Metadata.Add("x-amz-meta-original-file-name", filePath);
79-
80-
using (client = new AmazonS3Client(s3AccessKeyId,s3SecretAccessKey, s3Config))
74+
75+
using (client = new AmazonS3Client(s3AccessKeyId, s3SecretAccessKey, s3Config))
8176
{
8277
var response = client.PutObjectAsync(putRequest).Result;
8378
}
84-
8579
}
8680
catch (AmazonS3Exception e)
8781
{
@@ -96,8 +90,7 @@ public bool loadObjectToS3(String filePath, string name, string contentType, str
9690
, e.Message);
9791
}
9892

99-
return true;
100-
93+
return true;
10194
}
10295
}
103-
}
96+
}

‎log4net.config‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<log4net>
22
<root>
3-
<level value="ALL" />
3+
<level value="INFO" />
44
<appender-ref ref="console" />
55
<appender-ref ref="file" />
66
</root>

0 commit comments

Comments
 (0)