Skip to content

Commit 8d63eb7

Browse files
author
RaizeTheLimit
committed
feat(proto-log): configuarable options to save methods to JSON
1 parent 2a1206b commit 8d63eb7

5 files changed

Lines changed: 191 additions & 14 deletions

File tree

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ dist
55
/src/config/config.json
66
/.vs/
77
/.idea/
8+
/proto_samples/

src/config/example.config.json

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,11 @@
22
"default_port": 8081,
33
"trafficlight_identifier": "AwesomeProtoSender",
44
"redirect_to_golbat_url": null,
5-
"redirect_to_golbat_token": null
5+
"redirect_to_golbat_token": null,
6+
"sample_saving": {
7+
"enabled": true,
8+
"save_directory": "./proto_samples",
9+
"max_samples_per_method": 1,
10+
"endpoints": ["traffic", "golbat"]
11+
}
612
}

src/index.ts

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import http from "http";
22
import fs from "fs";
33
import { WebStreamBuffer, getIPAddress, handleData, moduleConfigIsAvailable, redirect_post_golbat } from "./utils";
44
import { decodePayload, decodePayloadTraffic } from "./parser/proto-parser";
5+
import SampleSaver from "./utils/sample-saver";
56

67
// try looking if config file exists...
78
let config = require("./config/example.config.json");
@@ -14,6 +15,9 @@ const incomingProtoWebBufferInst = new WebStreamBuffer();
1415
const outgoingProtoWebBufferInst = new WebStreamBuffer();
1516
const portBind = config["default_port"];
1617

18+
// Initialize sample saver
19+
const sampleSaver = config.sample_saving ? new SampleSaver(config.sample_saving) : null;
20+
1721
// server
1822
const httpServer = http.createServer(function (req, res) {
1923
let incomingData: Array<Buffer> = [];
@@ -49,6 +53,17 @@ const httpServer = http.createServer(function (req, res) {
4953
parsedData['contents'][i].request,
5054
"request"
5155
);
56+
const parsedResponseData = decodePayloadTraffic(
57+
parsedData['contents'][i].type,
58+
parsedData['contents'][i].payload,
59+
"response"
60+
);
61+
62+
// Save sample if enabled
63+
if (sampleSaver && parsedRequestData.length > 0 && parsedResponseData.length > 0) {
64+
sampleSaver.savePair(parsedRequestData[0], parsedResponseData[0], "golbat");
65+
}
66+
5267
if (typeof parsedRequestData === "string") {
5368
incomingProtoWebBufferInst.write({ error: parsedRequestData });
5469
} else {
@@ -57,11 +72,7 @@ const httpServer = http.createServer(function (req, res) {
5772
incomingProtoWebBufferInst.write(parsedObject);
5873
}
5974
}
60-
const parsedResponseData = decodePayloadTraffic(
61-
parsedData['contents'][i].type,
62-
parsedData['contents'][i].payload,
63-
"response"
64-
);
75+
6576
if (typeof parsedResponseData === "string") {
6677
outgoingProtoWebBufferInst.write({ error: parsedResponseData });
6778
} else {
@@ -85,10 +96,10 @@ const httpServer = http.createServer(function (req, res) {
8596
res.end("");
8697
if (Array.isArray(parsedData)) {
8798
for (let i = 0; i < parsedData.length; i++) {
88-
handleData(incomingProtoWebBufferInst, outgoingProtoWebBufferInst, identifier, parsedData[i])
99+
handleData(incomingProtoWebBufferInst, outgoingProtoWebBufferInst, identifier, parsedData[i], sampleSaver)
89100
}
90101
} else {
91-
handleData(incomingProtoWebBufferInst, outgoingProtoWebBufferInst, identifier, parsedData)
102+
handleData(incomingProtoWebBufferInst, outgoingProtoWebBufferInst, identifier, parsedData, sampleSaver)
92103
}
93104
});
94105
break;

src/utils/index.ts

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,24 @@ export function getIPAddress() {
3333
return '0.0.0.0';
3434
}
3535

36-
export function handleData(incoming: WebStreamBuffer, outgoing: WebStreamBuffer, identifier: any, parsedData: string) {
36+
export function handleData(incoming: WebStreamBuffer, outgoing: WebStreamBuffer, identifier: any, parsedData: string, sampleSaver?: any) {
3737
for (let i = 0; i < parsedData['protos'].length; i++) {
3838
const parsedRequestData = decodePayloadTraffic(
3939
parsedData['protos'][i].method,
4040
parsedData['protos'][i].request,
4141
"request"
4242
);
43+
const parsedResponseData = decodePayloadTraffic(
44+
parsedData['protos'][i].method,
45+
parsedData['protos'][i].response,
46+
"response"
47+
);
48+
49+
// Save sample if enabled
50+
if (sampleSaver && parsedRequestData.length > 0 && parsedResponseData.length > 0) {
51+
sampleSaver.savePair(parsedRequestData[0], parsedResponseData[0], "traffic");
52+
}
53+
4354
if (typeof parsedRequestData === "string") {
4455
incoming.write({ error: parsedRequestData });
4556
} else {
@@ -48,11 +59,7 @@ export function handleData(incoming: WebStreamBuffer, outgoing: WebStreamBuffer,
4859
incoming.write(parsedObject);
4960
}
5061
}
51-
const parsedResponseData = decodePayloadTraffic(
52-
parsedData['protos'][i].method,
53-
parsedData['protos'][i].response,
54-
"response"
55-
);
62+
5663
if (typeof parsedResponseData === "string") {
5764
outgoing.write({ error: parsedResponseData });
5865
} else {

src/utils/sample-saver.ts

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import fs from "fs";
2+
import path from "path";
3+
import { DecodedProto } from "../types";
4+
5+
interface SampleConfig {
6+
enabled: boolean;
7+
save_directory: string;
8+
max_samples_per_method: number;
9+
endpoints: string[];
10+
}
11+
12+
interface SavedSample {
13+
methodId: string;
14+
methodName: string;
15+
timestamp: string;
16+
request: any;
17+
response: any;
18+
}
19+
20+
interface SampleFile {
21+
filename: string;
22+
filepath: string;
23+
timestamp: number;
24+
methodId: string;
25+
}
26+
27+
class SampleSaver {
28+
private config: SampleConfig;
29+
private savedSamples: Map<string, SampleFile[]> = new Map();
30+
private saveDirectory: string;
31+
32+
constructor(config: SampleConfig) {
33+
this.config = config;
34+
this.saveDirectory = path.resolve(config.save_directory);
35+
this.initializeStorage();
36+
this.loadExistingSamples();
37+
}
38+
39+
private initializeStorage(): void {
40+
if (!this.config.enabled) return;
41+
42+
if (!fs.existsSync(this.saveDirectory)) {
43+
fs.mkdirSync(this.saveDirectory, { recursive: true });
44+
console.log(`Created sample storage directory: ${this.saveDirectory}`);
45+
}
46+
}
47+
48+
private loadExistingSamples(): void {
49+
if (!this.config.enabled) return;
50+
51+
if (fs.existsSync(this.saveDirectory)) {
52+
const files = fs.readdirSync(this.saveDirectory);
53+
54+
files.forEach(file => {
55+
const match = file.match(/^(\d+)_.*_(\d+)\.json$/);
56+
if (match) {
57+
const methodId = match[1];
58+
const timestamp = parseInt(match[2]);
59+
const filepath = path.join(this.saveDirectory, file);
60+
61+
const samples = this.savedSamples.get(methodId) || [];
62+
samples.push({
63+
filename: file,
64+
filepath: filepath,
65+
timestamp: timestamp,
66+
methodId: methodId
67+
});
68+
this.savedSamples.set(methodId, samples);
69+
}
70+
});
71+
72+
// Sort samples by timestamp for each method
73+
for (const [, samples] of this.savedSamples.entries()) {
74+
samples.sort((a, b) => a.timestamp - b.timestamp);
75+
}
76+
}
77+
}
78+
79+
private getTimestamp(): string {
80+
return new Date().toISOString();
81+
}
82+
83+
private deleteOldestSample(methodId: string): void {
84+
const samples = this.savedSamples.get(methodId);
85+
if (samples && samples.length > 0) {
86+
const oldest = samples.shift();
87+
if (oldest) {
88+
try {
89+
fs.unlinkSync(oldest.filepath);
90+
console.log(`Deleted oldest sample: ${oldest.filename}`);
91+
} catch (error) {
92+
console.error(`Failed to delete old sample: ${error}`);
93+
}
94+
}
95+
}
96+
}
97+
98+
private shouldSave(endpoint: string): boolean {
99+
if (!this.config.enabled) return false;
100+
if (!this.config.endpoints.includes(endpoint)) return false;
101+
return true;
102+
}
103+
104+
public async saveSample(request: DecodedProto, response: DecodedProto | null, endpoint: string): Promise<void> {
105+
if (!request || !request.methodId) return;
106+
if (!this.shouldSave(endpoint)) return;
107+
108+
const methodSamples = this.savedSamples.get(request.methodId) || [];
109+
110+
// If we've reached the max samples for this method, delete the oldest
111+
if (methodSamples.length >= this.config.max_samples_per_method) {
112+
this.deleteOldestSample(request.methodId);
113+
}
114+
115+
const sample: SavedSample = {
116+
methodId: request.methodId,
117+
methodName: request.methodName,
118+
timestamp: this.getTimestamp(),
119+
request: request.data,
120+
response: response ? response.data : null
121+
};
122+
123+
const safeMethodName = request.methodName.replace(/[^a-zA-Z0-9_-]/g, '_');
124+
const timestamp = Date.now();
125+
const filename = `${request.methodId}_${safeMethodName}_${timestamp}.json`;
126+
const filepath = path.join(this.saveDirectory, filename);
127+
128+
try {
129+
fs.writeFileSync(filepath, JSON.stringify(sample, null, 2));
130+
131+
// Update our tracking
132+
const samples = this.savedSamples.get(request.methodId) || [];
133+
samples.push({
134+
filename: filename,
135+
filepath: filepath,
136+
timestamp: timestamp,
137+
methodId: request.methodId
138+
});
139+
this.savedSamples.set(request.methodId, samples);
140+
141+
console.log(`Saved sample for method ${request.methodId} (${request.methodName}): ${filename}`);
142+
} catch (error) {
143+
console.error(`Failed to save sample: ${error}`);
144+
}
145+
}
146+
147+
public async savePair(request: DecodedProto, response: DecodedProto, endpoint: string): Promise<void> {
148+
await this.saveSample(request, response, endpoint);
149+
}
150+
}
151+
152+
export default SampleSaver;

0 commit comments

Comments
 (0)