daviddoji/summerschool_simtech_2023
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add material for testing (#11)

Browse Source 
* Add first draft material for testing

* added some first test stuff in julia

* add asserti error

* Rework intro slides

* added package

* Wrap up testing slides expect demo

* Smooth demo part

* Add statement on tests

* Remove previous python exercise file

---------

Co-authored-by: behinger (s-ccs 001) <benedikt.ehinger@vis.uni-stuttgart.de>
This commit is contained in:
Benjamin Uekermann 2023-10-06 15:54:38 +02:00 committed by GitHub
parent e6f7ff3ed1
commit 937a8a9481
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 448 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
material/1_mon/envs/MyStatsPackage/Manifest.toml Normal file
View File

@ -0,0 +1,36 @@
# This file is machine-generated - editing it directly is not advised
julia_version = "1.9.2"
manifest_format = "2.0"
project_hash = "ead88d718d76311b89d27bda5ebf9eab16288dd1"
[[deps.Distributed]]
deps = ["Random", "Serialization", "Sockets"]
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
[[deps.ProgressMeter]]
deps = ["Distributed", "Printf"]
git-tree-sha1 = "00099623ffee15972c16111bcf84c58a0051257c"
uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
version = "1.9.0"
[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"
[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
[[deps.Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

7
material/1_mon/envs/MyStatsPackage/Project.toml Normal file
View File

@ -0,0 +1,7 @@
name = "MyStatsPackage"
uuid = "867243ab-1459-4018-9627-69ff6fe7dbfd"
authors = ["behinger (s-ccs 001) <benedikt.ehinger@vis.uni-stuttgart.de>"]
version = "0.1.0"
[deps]
ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"

10
material/1_mon/envs/MyStatsPackage/src/MyStatsPackage.jl Normal file
View File

@ -0,0 +1,10 @@
module MyStatsPackage
include("../../../firststeps/statistic_functions.jl")
export rse_mean
export rse_std
export rse_sum
export rse_tstat
end # module MyStatsPackage

13
material/1_mon/firststeps/statistic_functions.jl
View File

@ -1,6 +1,6 @@
#---
using ProgressMeter
import Base: length
function rse_sum(x)
s = 0
@showprogress for k = eachindex(x)
@ -32,21 +32,22 @@ end
rse_tstat(2:3) == 5
#---
struct StatResult
x::Vector
n::Int32
std::Float64
tvalue::Float64
end
length(s::StatResult) = s.n
Base.length(s::StatResult) = s.n
StatResult(x) = StatResult(x,length(x))
StatResult(x,n) = StatResult(x,n,rse_std(x))
StatResult(x,n,std) = StatResult(x,n,)
StatResult(x,n,std) = StatResult(x,n,std,rse_tstat(x;σ=std))
mystatresult(10,500.) # <1>
StatResult([10,500.]) # <1>
function tstat(x) # <2> generate a function returning our new type
return mystatresult(length(x),rse_tstat(x))
function tstat(x)
return StatResult(length(x),rse_tstat(x))
end

7
material/2_tue/testing/MyTestPackage/Manifest.toml Normal file
View File

@ -0,0 +1,7 @@
# This file is machine-generated - editing it directly is not advised
julia_version = "1.9.2"
manifest_format = "2.0"
project_hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
[deps]

4
material/2_tue/testing/MyTestPackage/Project.toml Normal file
View File

@ -0,0 +1,4 @@
name = "MyTestPackage"
uuid = "5a49bdff-dc99-4c0a-8062-dec1d6c69516"
authors = ["behinger (s-ccs 001) <benedikt.ehinger@vis.uni-stuttgart.de>"]
version = "0.1.0"

6
material/2_tue/testing/MyTestPackage/src/MyTestPackage.jl Normal file
View File

@ -0,0 +1,6 @@
module MyTestPackage
include("find.jl")
export find_max
export find_mean
end # module MyTestPackage

16
material/2_tue/testing/MyTestPackage/src/find.jl Normal file
View File

@ -0,0 +1,16 @@
function find_max(x::AbstractVector)
@assert all(!isnan,x)
currentmax = x[1]
for a = eachindex(x)
if x[a] > currentmax
currentmax = x[a]
end
end
return currentmax
end
function find_mean(x::AbstractVector)
@assert all(!isnan,x)
return sum(x)./length(x)
end

40
material/2_tue/testing/MyTestPackage/test/Manifest.toml Normal file
View File

@ -0,0 +1,40 @@
# This file is machine-generated - editing it directly is not advised
julia_version = "1.9.2"
manifest_format = "2.0"
project_hash = "c34966543d8e8dc43716ca8888b0447b9c23c175"
[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
[[deps.Random]]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"
[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
[[deps.StableRNGs]]
deps = ["Random", "Test"]
git-tree-sha1 = "3be7d49667040add7ee151fefaf1f8c04c8c8276"
uuid = "860ef19b-820b-49d6-a774-d7a799459cd3"
version = "1.0.0"
[[deps.Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

3
material/2_tue/testing/MyTestPackage/test/Project.toml Normal file
View File

@ -0,0 +1,3 @@
[deps]
StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

34
material/2_tue/testing/MyTestPackage/test/find.jl Normal file
View File

@ -0,0 +1,34 @@
@testset "unit tests" begin
@testset "find_max" begin
@test find_max([1,2,3]) == 3
@test find_max([0,0,0]) == 0
@test_throws AssertionError find_max([NaN,3,2])
end
@testset "find_mean" begin
@test find_mean([1,2,3]) == 2
@test find_mean([1,3,6]) ≈ 3.333 atol=1e-3
end
end
@testset "integration test" begin
data1 = [43, 32, 167, 18, 1, 209]
data2 = [3, 13, 33, 23, 498]
# Expected result
expected_mean_of_max = 353.5
maximum1 = find_max(data1)
maximum2 = find_max(data2)
# Actual result
actual_mean_of_max = find_mean([maximum1, maximum2])
# Test
@test actual_mean_of_max == expected_mean_of_max
end

7
material/2_tue/testing/MyTestPackage/test/runtests.jl Normal file
View File

@ -0,0 +1,7 @@
using MyTestPackage # here it is ok to use, don't put it in your "debug"-convenience setup.jl
include("setup.jl")
@testset "find" begin
include("find.jl")
end

2
material/2_tue/testing/MyTestPackage/test/setup.jl Normal file
View File

@ -0,0 +1,2 @@
using Test
using StableRNGs

269
material/2_tue/testing/slides.qmd Normal file
View File

@ -0,0 +1,269 @@
---
format: revealjs
---
# Learning Goals
- Justify the effort of developing tests to some extent
- Get to know a few common terms of testing
- Work with the Julia unit testing package `Test.jl`
Material is taken and modified, on the one hand, from the [SSE lecture](https://github.com/Simulation-Software-Engineering/Lecture-Material), which builds partly on the [py-rse book](https://merely-useful.tech/py-rse), and, on the other hand, from the [Test.jl docs](https://docs.julialang.org/en/v1/stdlib/Test/).
---
# 1. General Introduction to Testing
---
## What is Testing?
- Smelling old milk before using it!
- A way to determine if a software is not producing reliable results and if so, what is the reason.
- Manual testing vs. automated testing.
---
## Why Should you Test your Software?
- Improve software reliability and reproducibility.
- Make sure that changes (bugfixes, new features) do not affect other parts of software.
- Generally all software is better off being tested regularly. Possible exceptions are very small codes with single users.
- Ensure that a released version of a software actually works.
---
## Nomenclature in Software Testing
- **Fixture**: preparatory set for testing.
- **Actual result**: what the code produces when given the fixture.
- **Expected result**: what the actual result is compared to.
- **Test coverage**: how much of the code do tests touch in one run.
---
## Some Ways to Test Software
- Assertions
- Unit testing
- Integration testing
- Regression testing
---
## Assertions
- Principle of *defensive programming*.
- Nothing happens when an assertion is true; throws error when false.
- Types of assertion statements:
- Precondition
- Postcondition
- Invariant
- A basic but powerful tool to test a software on-the-go.
- Assertion statement syntax in Python
```julia
@assert condition "message"
```
---
## Unit Testing
- Catching errors with assertions is good but preventing them is better!
- A *unit* is a single function in one situation.
- A situation is one amongst many possible variations of input parameters.
- User creates the expected result manually.
- Fixture is the set of inputs used to generate an actual result.
- Actual result is compared to the expected result by `@test`.
---
## Integration Testing
- Test whether several units work in conjunction.
- *Integrate* units and test them together in an *integration* test.
- Often more complicated than a unit test and has more test coverage.
- A fixture is used to generate an actual result.
- Actual result is compared to the expected result by `@test`.
---
## Regression Testing
- Generating an expected result is not possible in some situations.
- Compare the current actual result with a previous actual result.
- No guarantee that the current actual result is correct.
- Risk of a bug being carried over indefinitely.
- Main purpose is to identify changes in the current state of the code with respect to a past state.
---
## Test Coverage
- Coverage is the amount of code a test touches in one run.
- Aim for high test coverage.
- There is a trade-off: high test coverage vs. effort in test development
---
## Comparing Floating-point Variables
- Very often quantities in math software are `float` / `double`.
- Such quantities cannot be compared to exact values, an approximation is necessary.
- Comparison of floating point variables needs to be done to a certain tolerance.
```julia
@test 1 ≈ 0.999999 rtol=1e-5
```
- Get `≈` by Latex `\approx` + TAB
---
## Test-driven Development (TDD)
- Principle is to write a test and then write a code to fulfill the test.
- Advantages:
- In the end user ends up with a test alongside the code.
- Eliminates confirmation bias of the user.
- Writing tests gives clarity on what the code is supposed to do.
- Disadvantage: known to not improve productivity.
---
## Checking-driven Development (CDD)
- Developer performs spot checks; sanity checks at intermediate stages
- Math software often has heuristics which are easy to determine.
- Keep performing same checks at different stages of development to ensure the code works.
---
## Verifying a Test
- Test written as part of a bug-fix:
- Reproduce the bug in the test by ensuring that the test fails.
- Fix the bug.
- Rerun the test to ensure that it passes.
- Test written to increase code coverage:
- Make sure that the first iteration of the test passes.
- Try introducing a small fixable bug in the code to verify if the test fails.
---
# 2. Unit Testing in Julia with Test.jl
---
## Setup of Tests.jl
- Standard library to write and manage tests, `using Test`
- Standardized folder structure:
```
├── Manifest.toml
├── Project.toml
├── src/
└── test
├── Manifest.toml
├── Project.toml
├── runtests.jl
└── setup.jl
```
- Singular `test` vs plural `runtests.jl`
- `setup.jl` for all `using XYZ` statements, included in `runtests.jl`
- Additional packages either in `[extra] section` of `./Project.toml` or in a new `./test/Project.toml` environment
- In case of the latter: Do not add the package itself to the `./test/Project.toml`
---
## Run Tests
Various options:
- Directly call `runtests.jl` TODO?
- From Pkg-Manager `]test` when root project is activated
---
## Implement and Structure Tests
- `@test expr`: Test whether expression `expr` is true
- `@test expr broken=true`: Explicitly mark test as broken
- `@test_throws exception expr`: Test whether expression `expr` throws `exception` (test unhappy path)
```julia
julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2]
Test Passed
Thrown: DimensionMismatch
```
- `@testset`: Structure tests
```julia
julia> @testset "trigonometric identities" begin
θ = 2/3*π
@test sin(-θ) ≈ -sin(θ)
@test cos(-θ) ≈ cos(θ)
end;
```
- `@testset for ... end`: Test in loop
---
## Further Reading and Watching
- [Research Software Engineering with Python - Chapter 11: Testing Software](https://merely-useful.tech/py-rse/testing.html)
- [HiRSE-Summer of Testing Part 2b: "Testing with Julia" by Nils Niggemann](https://www.youtube.com/watch?v=gSMKNbZOpZU)
- [Official documentation of Test.jl](https://docs.julialang.org/en/v1/stdlib/Test/)
# 3. Test.jl Demo
We use [`MyTestPackage`](https://github.com/s-ccs/summerschool_simtech_2023/tree/main/material/2_tue/testing/MyTestPackage), which looks as follows:
```
├── Manifest.toml
├── Project.toml
├── src
│   ├── find.jl
│   └── MyTestPackage.jl
└── test
├── find.jl
├── Manifest.toml
├── Project.toml
├── runtests.jl
└── setup.jl
```
- Look at `MyTestPackage.jl` and `find.jl`: We have two functions `find_max` and `find_mean`, which calculate the maximum and mean of all elements of a `::AbstractVector`.
- Assertions were added to check for `NaN` values
- Look at `runtests.jl`:
- TODO: Why do we need `using MyTestPackage`?
- We include dependencies via `setup.jl`: `Test` and `StableRNG`.
- Testset "find"
- Look at `find.jl`
- Unit tests for `find_max` and `find_mean`
- `test_throws` to test unhappy path
- Test with absolute tolerance
- Integration test, which tests combination of both methods
- Run tests:
```
]activate .
]test
```
---
# 4. Exercise
Write tests for your own statistics package 😊