Skip to content

Commit 770647e

Browse files
patr1ck-mpatr1ck-m
authored
Merge pull request patr1ck-m#13 from patr1ck-m/feature/simulated_data_input
Add statistical distribution functions
2 parents adb5765 + e1f31ee commit 770647e

4 files changed

Lines changed: 176 additions & 16 deletions

File tree

‎src/lib/csv.effekt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ import string
2121
--- The following part is written by myself ---
2222
-----------------------------------------------
2323
*/
24-
24+
extern def trim(s: String) at {}: String =
25+
js "${s}.trim()"
2526

2627
def collectList[T, R]{ body: => R / { emit[T] } }: (R, List[T]) = {
2728
var lst: List[T] = Nil()
@@ -219,7 +220,7 @@ def getColumnOf[R](columnName: String) { csv: => R / CsvBuilder }: R / {emit[Str
219220
val ret = try r() with RowBuilder {
220221
def cell(s) = {
221222
currentColumn = currentColumn + 1
222-
if (columnId is None() and s == columnName) {
223+
if (columnId is None() and s.trim() == columnName.trim()) {
223224
columnId = Some(currentColumn)
224225
} else if (columnId is Some(c) and c == currentColumn) {
225226
do emit(s)

‎src/lib/stream_input.effekt‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,24 @@ def csvFeedStrPull(csvStr: String, columnName: String, delayMs: Int) {body: () =
7979
pushToPullStream[Double](100) { csvFeedStr(csvStr, columnName, delayMs) } { body() }
8080
}
8181

82+
def normalDistributedPull(mu: Double, sigma: Double) {body: () => Unit / { read[Double] } }: Unit / { } = {
83+
try body() with read[Double] {
84+
resume { internal::generateNormalDistributedValue(mu, sigma) }
85+
}
86+
}
87+
88+
def exponentialDistributedPull(lambda: Double) {body: () => Unit / { read[Double] } }: Unit / { } = {
89+
try body() with read[Double] {
90+
resume { internal::generateExponentialDistributedValue(lambda) }
91+
}
92+
}
93+
94+
def uniformDistributedPull(a: Double, b: Double) {body: () => Unit / { read[Double] } }: Unit / { } = {
95+
try body() with read[Double] {
96+
resume { internal::generateUniformDistributedValue(a, b) }
97+
}
98+
}
99+
82100

83101
namespace examples {
84102
def example1(): Unit = {
@@ -115,7 +133,51 @@ namespace examples {
115133
}
116134
}
117135

136+
def example4(): Unit = {
137+
normalDistributedPull(0.0, 1.0) {
138+
effekt::each(1, 11) { (n) =>
139+
try {
140+
val v = do read[Double]()
141+
println("Normal distributed value: " ++ v.show())
142+
} with stop {
143+
println("This should never be called since the stream is infinite.")
144+
}
145+
}
146+
}
147+
}
148+
149+
118150
def main(): Unit = {
119151
example2()
120152
}
121153
}
154+
155+
156+
namespace internal {
157+
def generateNormalDistributedValue(mu: Double, sigma: Double): Double = {
158+
var z = 0.0
159+
loop { {l} =>
160+
val u1 = random()
161+
val u2 = random()
162+
val v1 = 2.0 * u1 - 1.0
163+
val v2 = 2.0 * u2 - 1.0
164+
val w = v1 * v1 + v2 * v2
165+
if (w < 1.0) {
166+
val y = sqrt((-2.0 * log(w)) / w)
167+
z = v1 * y
168+
l.break()
169+
}
170+
}
171+
return mu + z * sigma
172+
}
173+
174+
def generateExponentialDistributedValue(lambda: Double): Double = {
175+
val u = random()
176+
return (-1.0 / lambda) * log(u)
177+
}
178+
179+
def generateUniformDistributedValue(a: Double, b: Double): Double = {
180+
val u = random()
181+
return a + (b - a) * u
182+
}
183+
}

‎src/test/lib.effekt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ def assertDoNotRaise[E] {block: () => Unit / Exception[E]} = {
6666
block()
6767
} with Exception[E] {
6868
def raise(e: E, msg: String) = {
69-
assertTrue(false, "Expected exception not to raise but was raised")
69+
assertTrue(false, "Expected exception not to raise but was raised: " ++ msg)
7070
}
7171
}
7272
}

‎src/test/stream_input_test.effekt‎

Lines changed: 110 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,117 @@ def testSuite() = suite("stream_input_csv") {
2323
}
2424
}
2525

26-
// test("raises IOError on missing file") {
27-
// with assertDoRaise[IOError]
28-
// with assertDoNotRaise[WrongFormat]
29-
// assertEmits[Double]([]) { (a: Double, b: Double) => a == b } {
30-
// stream_input::csvFeed("src/test/data/missing_file.csv", "value", 0)
31-
// }
32-
// }
26+
test("raises IOError on missing file") {
27+
with assertDoRaise[IOError]
28+
with assertDoNotRaise[WrongFormat]
29+
assertEmits[Double]([]) { (a: Double, b: Double) => a == b } {
30+
stream_input::csvFeed("missing_file.csv", "value", 0)
31+
}
32+
}
3333

34-
// test("read non existing file") {
35-
// assertDoRaise[IOError] {
36-
// readFile("non_existing_file") // should raise IOError but backend error occurs
37-
// ()
38-
// }
39-
// }
34+
test("read non existing file") {
35+
assertDoRaise[IOError] {
36+
readFile("non_existing_file") // should raise IOError but backend error occurs
37+
()
38+
}
39+
}
40+
41+
42+
// ----------------------------------------------------------------------------------------------------------
43+
// The following tests are statistical and may fail occasionally due to randomness (intentional flaky tests!)
44+
// When the test fails, it is an indication that there may be an issue with the distribution generator
45+
// ----------------------------------------------------------------------------------------------------------
46+
47+
test("read normal distributed values") {
48+
val mu = 0.0
49+
val sigma = 1.0
50+
val numSamples = 100000
51+
val acceptedOutliers = 100 // Allow some outliers due to randomness
52+
var samplesRead = 0
53+
var samplesOutliers = 0
54+
var sampleSum = 0.0
55+
var sampleSumSq = 0.0
56+
57+
normalDistributedPull(mu, sigma) {
58+
while (samplesRead < numSamples) {
59+
try {
60+
val v = do read[Double]()
61+
// Check that the value is within a reasonable range (e.g., within 5 standard deviations)
62+
if (not(v >= mu - 5.0 * sigma && v <= mu + 5.0 * sigma))
63+
samplesOutliers = samplesOutliers + 1
64+
samplesRead = samplesRead + 1
65+
sampleSum = sampleSum + v
66+
sampleSumSq = sampleSumSq + v * v
67+
} with stop {
68+
assertTrue(false, "Infinite stream stopped unexpectedly")
69+
}
70+
}
71+
}
72+
assertTrue(samplesOutliers <= acceptedOutliers, "Statistical test failed: too many outliers. This may indicate an issue with the normal distribution generator but could also be due to random chance.")
73+
val sampleMean = sampleSum / toDouble(numSamples)
74+
val sampleVariance = (sampleSumSq / toDouble(numSamples)) - (sampleMean * sampleMean)
75+
val sampleStdDev = sqrt(sampleVariance)
76+
assertTrue(abs(sampleMean - mu) < 0.2, "Statistical test failed: sample mean deviates significantly from expected mean. This may indicate an issue with the normal distribution generator but could also be due to random chance.")
77+
assertTrue(abs(sampleStdDev - sigma) < 0.2, "Statistical test failed: sample standard deviation deviates significantly from expected standard deviation. This may indicate an issue with the normal distribution generator but could also be due to random chance.")
78+
}
79+
80+
test("read exponential distributed values") {
81+
val lambda = 1.0
82+
val numSamples = 100000
83+
val acceptedOutliers = 100 // Allow some outliers due to randomness
84+
var samplesRead = 0
85+
var samplesOutliers = 0
86+
var sampleSum = 0.0
87+
88+
exponentialDistributedPull(lambda) {
89+
while (samplesRead < numSamples) {
90+
try {
91+
val v = do read[Double]()
92+
93+
assertTrue(v >= 0.0, "Exponential distributed value should be non-negative")
94+
if (v > 10.0 / lambda)
95+
samplesOutliers = samplesOutliers + 1
96+
samplesRead = samplesRead + 1
97+
sampleSum = sampleSum + v
98+
} with stop {
99+
assertTrue(false, "Infinite stream stopped unexpectedly")
100+
}
101+
}
102+
}
103+
assertTrue(samplesOutliers <= acceptedOutliers, "Statistical test failed: too many outliers. This may indicate an issue with the exponential distribution generator but could also be due to random chance.")
104+
val sampleMean = sampleSum / toDouble(numSamples)
105+
val expectedMean = 1.0 / lambda
106+
assertTrue(abs(sampleMean - expectedMean) < 0.2, "Statistical test failed: sample mean deviates significantly from expected mean. This may indicate an issue with the exponential distribution generator but could also be due to random chance.")
107+
}
108+
109+
test("read uniform distributed values") {
110+
val a = -5.0
111+
val b = 5.0
112+
val numSamples = 100000
113+
val acceptedOutliers = 100 // Allow some outliers due to randomness
114+
var samplesRead = 0
115+
var samplesOutliers = 0
116+
var sampleSum = 0.0
117+
118+
uniformDistributedPull(a, b) {
119+
while (samplesRead < numSamples) {
120+
try {
121+
val v = do read[Double]()
122+
123+
if (v < a || v > b)
124+
samplesOutliers = samplesOutliers + 1
125+
samplesRead = samplesRead + 1
126+
sampleSum = sampleSum + v
127+
} with stop {
128+
assertTrue(false, "Infinite stream stopped unexpectedly")
129+
}
130+
}
131+
}
132+
assertTrue(samplesOutliers <= acceptedOutliers, "Statistical test failed: too many outliers. This may indicate an issue with the uniform distribution generator but could also be due to random chance.")
133+
val sampleMean = sampleSum / toDouble(numSamples)
134+
val expectedMean = (a + b) / 2.0
135+
assertTrue(abs(sampleMean - expectedMean) < 0.2, "Statistical test failed: sample mean deviates significantly from expected mean. This may indicate an issue with the uniform distribution generator but could also be due to random chance.")
136+
}
40137
}
41138

42139
def main() = {

0 commit comments

Comments
 (0)