daviddoji/JuliaForDataAnalysis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reorganize chapters

Browse Source
This commit is contained in:
Bogumił Kamiński 2022-02-08 20:58:33 +01:00
parent c44c4f1609
commit 55045fc5dd
12 changed files with 1662 additions and 1596 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
ch02.jl
View File

@ -329,142 +329,3 @@ function fun6()
end
end
fun6()
# Code from section 2.6
methods(cd)
sum isa Function
typeof(sum)
typeof(sum) == Function
supertype(typeof(sum))
function traverse(T)
println(T)
T == Any || traverse(supertype(T))
return nothing
end
traverse(Int64)
function print_subtypes(T, indent_level=0)
println(" " ^ indent_level, T)
for S in subtypes(T)
print_subtypes(S, indent_level + 2)
end
return nothing
end
print_subtypes(Integer)
traverse(typeof([1.0, 2.0, 3.0]))
traverse(typeof(1:3))
AbstractVector
typejoin(typeof([1.0, 2.0, 3.0]), typeof(1:3))
# Code from section 2.7
fun(x) = println("unsupported type")
fun(x::Number) = println("a number was passed")
fun(x::Float64) = println("a Float64 value")
methods(fun)
fun("hello!")
fun(1)
fun(1.0)
bar(x, y) = "no numbers passed"
bar(x::Number, y) = "first argument is a number"
bar(x, y::Number) = "second argument is a number"
bar("hello", "world")
bar(1, "world")
bar("hello", 2)
bar(1, 2)
bar(x::Number, y::Number) = "both arguments are numbers"
bar(1, 2)
methods(bar)
function winsorized_mean(x::AbstractVector, k::Integer)
k >= 0 || throw(ArgumentError("k must be non-negative"))
length(x) > 2 * k || throw(ArgumentError("k is too large"))
y = sort!(collect(x))
for i in 1:k
y[i] = y[k + 1]
y[end - i + 1] = y[end - k]
end
return sum(y) / length(y)
end
winsorized_mean([8, 3, 1, 5, 7], 1)
winsorized_mean(1:10, 2)
winsorized_mean(1:10, "a")
winsorized_mean(10, 1)
winsorized_mean(1:10, -1)
winsorized_mean(1:10, 5)
# Code from section 2.8
module ExampleModule
function example()
println("Hello")
end
end # ExampleModule
import Statistics
x = [1, 2, 3]
mean(x)
Statistics.mean(x)
using Statistics
mean(x)
# start a fresh Julia session before running this code
mean = 1
using Statistics
mean
# start a fresh Julia session before running this code
using Statistics
mean([1, 2, 3])
mean = 1
# start a fresh Julia session before running this code
using Statistics
mean = 1
mean([1, 2, 3])
# start a fresh Julia session before running this code
using Statistics
using StatsBase
?winsor
mean(winsor([8, 3, 1, 5, 7], count=1))
# Code from section 2.9
@time 1 + 2
@time(1 + 2)
@assert 1 == 2 "1 is not equal 2"
@assert(1 == 2, "1 is not equal 2")
@macroexpand @assert(1 == 2, "1 is not equal 2")
@macroexpand @time 1 + 2
# before running these codes
# define the winsorized_mean function using the code from section 2.7
using BenchmarkTools
x = rand(10^6);
@benchmark winsorized_mean($x, 10^5)
using Statistics, StatsBase
@benchmark mean(winsor($x; count=10^5))
@edit winsor(x, count=10^5)

475
ch03.jl
View File

@ -2,358 +2,141 @@
# Codes for chapter 3
# Code for listing 3.1
# Code from section 3.1
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
methods(cd)
# Code for checking size of a matrix
sum isa Function
size(aq)
size(aq, 1)
size(aq, 2)
typeof(sum)
typeof(sum) == Function
# Code comparing tuple to a vector
supertype(typeof(sum))
v = [1, 2, 3]
t = (1, 2, 3)
v[1]
t[1]
v[1] = 10
v
t[1] = 10
function traverse(T)
println(T)
T == Any || traverse(supertype(T))
return nothing
end
traverse(Int64)
# Code for figure 3.2
function print_subtypes(T, indent_level=0)
println(" " ^ indent_level, T)
for S in subtypes(T)
print_subtypes(S, indent_level + 2)
end
return nothing
end
print_subtypes(Integer)
traverse(typeof([1.0, 2.0, 3.0]))
traverse(typeof(1:3))
AbstractVector
typejoin(typeof([1.0, 2.0, 3.0]), typeof(1:3))
# Code from section 3.2
fun(x) = println("unsupported type")
fun(x::Number) = println("a number was passed")
fun(x::Float64) = println("a Float64 value")
methods(fun)
fun("hello!")
fun(1)
fun(1.0)
bar(x, y) = "no numbers passed"
bar(x::Number, y) = "first argument is a number"
bar(x, y::Number) = "second argument is a number"
bar("hello", "world")
bar(1, "world")
bar("hello", 2)
bar(1, 2)
bar(x::Number, y::Number) = "both arguments are numbers"
bar(1, 2)
methods(bar)
function winsorized_mean(x::AbstractVector, k::Integer)
k >= 0 || throw(ArgumentError("k must be non-negative"))
length(x) > 2 * k || throw(ArgumentError("k is too large"))
y = sort!(collect(x))
for i in 1:k
y[i] = y[k + 1]
y[end - i + 1] = y[end - k]
end
return sum(y) / length(y)
end
winsorized_mean([8, 3, 1, 5, 7], 1)
winsorized_mean(1:10, 2)
winsorized_mean(1:10, "a")
winsorized_mean(10, 1)
winsorized_mean(1:10, -1)
winsorized_mean(1:10, 5)
# Code from section 3.3
module ExampleModule
function example()
println("Hello")
end
end # ExampleModule
import Statistics
x = [1, 2, 3]
mean(x)
Statistics.mean(x)
using Statistics
mean(x)
# start a fresh Julia session before running this code
mean = 1
using Statistics
mean
# start a fresh Julia session before running this code
using Statistics
mean([1, 2, 3])
mean = 1
# start a fresh Julia session before running this code
using Statistics
mean = 1
mean([1, 2, 3])
# start a fresh Julia session before running this code
using Statistics
using StatsBase
?winsor
mean(winsor([8, 3, 1, 5, 7], count=1))
# Code from section 3.4
@time 1 + 2
@time(1 + 2)
@assert 1 == 2 "1 is not equal 2"
@assert(1 == 2, "1 is not equal 2")
@macroexpand @assert(1 == 2, "1 is not equal 2")
@macroexpand @time 1 + 2
# before running these codes
# define the winsorized_mean function using the code from section 3.1
using BenchmarkTools
@benchmark (1, 2, 3)
@benchmark [1, 2, 3]
x = rand(10^6);
@benchmark winsorized_mean($x, 10^5)
using Statistics, StatsBase
@benchmark mean(winsor($x; count=10^5))
# Code for section 3.1.2
using Statistics
mean(aq; dims=1)
std(aq; dims=1)
map(mean, eachcol(aq))
map(std, eachcol(aq))
map(eachcol(aq)) do col
mean(col)
end
[mean(col) for col in eachcol(aq)]
[std(col) for col in eachcol(aq)]
# Code for section 3.1.3
[mean(aq[:, j]) for j in axes(aq, 2)]
[std(aq[:, j]) for j in axes(aq, 2)]
axes(aq, 2)
?Base.OneTo
[mean(view(aq, :, j)) for j in axes(aq, 2)]
[std(@view aq[:, j]) for j in axes(aq, 2)]
# Code for section 3.1.4
using BenchmarkTools
x = ones(10^7, 10)
@benchmark [mean(@view $x[:, j]) for j in axes($x, 2)]
@benchmark [mean($x[:, j]) for j in axes($x, 2)]
@benchmark mean($x, dims=1)
# Code for section 3.1.5
[cor(aq[:, i], aq[:, i+1]) for i in 1:2:7]
collect(1:2:7)
# Code for section 3.1.6
y = aq[:, 2]
X = [ones(11) aq[:, 1]]
X \ y
[[ones(11) aq[:, i]] \ aq[:, i+1] for i in 1:2:7]
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
error = y - prediction
SS_res = sum(v -> v ^ 2, error)
mean_y = mean(y)
SS_tot = sum(v -> (v - mean_y) ^ 2, y)
return 1 - SS_res / SS_tot
end
[R²(aq[:, i], aq[:, i+1]) for i in 1:2:7]
?²
# Code for section 3.1.7
using Plots
scatter(aq[:, 1], aq[:, 2]; legend=false)
plot(scatter(aq[:, 1], aq[:, 2]; legend=false),
scatter(aq[:, 3], aq[:, 4]; legend=false),
scatter(aq[:, 5], aq[:, 6]; legend=false),
scatter(aq[:, 7], aq[:, 8]; legend=false))
plot([scatter(aq[:, i], aq[:, i+1]; legend=false)
for i in 1:2:7]...)
# Code for section 3.2
two_standard = Dict{Int, Int}()
for i in [1, 2, 3, 4, 5, 6]
for j in [1, 2, 3, 4, 5, 6]
s = i + j
if haskey(two_standard, s)
two_standard[s] += 1
else
two_standard[s] = 1
end
end
end
two_standard
keys(two_standard)
values(two_standard)
using Plots
scatter(collect(keys(two_standard)), collect(values(two_standard));
legend=false, xaxis=2:12)
all_dice = [[1, x2, x3, x4, x5, x6]
for x2 in 2:11
for x3 in x2:11
for x4 in x3:11
for x5 in x4:11
for x6 in x5:11]
for d1 in all_dice, d2 in all_dice
test = Dict{Int, Int}()
for i in d1, j in d2
s = i + j
if haskey(test, s)
test[s] += 1
else
test[s] = 1
end
end
if test == two_standard
println(d1, " ", d2)
end
end
# Code for section 3.3
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
dataset1 = (x=aq[:, 1], y=aq[:, 2])
dataset1[1]
dataset1.x
# Code for listing 3.2
data = (set1=(x=aq[:, 1], y=aq[:, 2]),
set2=(x=aq[:, 3], y=aq[:, 4]),
set3=(x=aq[:, 5], y=aq[:, 6]),
set4=(x=aq[:, 7], y=aq[:, 8]))
# Code for section 3.3.2
using Statistics
map(s -> mean(s.x), data)
map(s -> cor(s.x, s.y), data)
using GLM
model = lm(@formula(y ~ x), data.set1)
r2(model)
# Code for section 3.3.3
model.mm
x = [3, 1, 2]
sort(x)
x
sort!(x)
x
empty_field!(nt, i) = empty!(nt[i])
nt = (dict = Dict("a" => 1, "b" => 2), int=10)
empty_field!(nt, 1)
nt
# Code for section 3.4.1
x = [1 2 3]
y = [1, 2, 3]
x * y
a = [1, 2, 3]
b = [4, 5, 6]
a * b
a .* b
map(*, a, b)
[a[i] * b[i] for i in eachindex(a, b)]
eachindex(a, b)
eachindex([1, 2, 3], [4, 5])
map(*, [1, 2, 3], [4, 5])
[1, 2, 3] .* [4, 5]
# Code for section 3.4.2
[1, 2, 3] .* [4]
[1, 2, 3] .^ 2
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] .* [1 2 3 4 5 6 7 8 9 10]
["x", "y", "z"] .=> [sum minimum maximum]
abs.([1, -2, 3, -4])
abs([1, 2, 3])
string(1, 2, 3)
string.("x", 1:10)
f(i::Int) = string("got integer ", i)
f(s::String) = string("got string ", s)
f.([1, "1"])
# Code for section 3.4.3
in(1, [1, 2, 3])
in(4, [1, 2, 3])
in([1, 3, 5, 7, 9], [1, 2, 3, 4])
in.([1, 3, 5, 7, 9], [1, 2, 3, 4])
in.([1, 3, 5, 7, 9], Ref([1, 2, 3, 4]))
# Code for section 3.4.4
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
using Statistics
mean.(eachcol(aq))
mean(eachcol(aq))
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
error = y - prediction
SS_res = sum(v -> v ^ 2, error)
mean_y = mean(y)
SS_tot = sum(v -> (v - mean_y) ^ 2, y)
return 1 - SS_res / SS_tot
end
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
SS_res = sum((y .- prediction) .^ 2)
SS_tot = sum((y .- mean(y)) .^ 2)
return 1 - SS_res / SS_tot
end
# Code for section 3.5
[]
Dict()
Float64[1, 2, 3]
Dict{UInt8, Float64}(0 => 0, 1 => 1)
UInt32(200)
Real[1, 1.0, 0x3]
v1 = Any[1, 2, 3]
eltype(v1)
v2 = Float64[1, 2, 3]
eltype(v2)
v3 = [1, 2, 3]
eltype(v2)
d1 = Dict()
eltype(d1)
d2 = Dict(1 => 2, 3 => 4)
eltype(d2)
p = 1 => 2
typeof(p)
# Code for section 3.5.1
[1, 2, 3] isa AbstractVector{Int}
[1, 2, 3] isa AbstractVector{Real}
AbstractVector{<:Real}
# Code for section 3.5.2
using Statistics
function ourcov(x::AbstractVector{<:Real},
y::AbstractVector{<:Real})
len = length(x)
@assert len == length(y) > 0
return sum((x .- mean(x)) .* (y .- mean(y))) / (len - 1)
end
ourcov(1:4, [1.0, 3.0, 2.0, 4.0])
cov(1:4, [1.0, 3.0, 2.0, 4.0])
ourcov(1:4, Any[1.0, 3.0, 2.0, 4.0])
x = Any[1, 2, 3]
identity.(x)
y = Any[1, 2.0]
identity.(y)
@edit winsor(x, count=10^5)

224
ch04.jl
View File

@ -1,224 +0,0 @@
# Bogumił Kamiński, 2022
# Codes for chapter 4
# Code for listing 4.1
import Downloads
Downloads.download("https://raw.githubusercontent.com/" *
"sidooms/MovieTweetings/" *
"44c525d0c766944910686c60697203cda39305d6/" *
"snapshots/10K/movies.dat",
"movies.dat")
# Code for string interpolation examples
x = 10
"I have $x apples"
"I have \$100."
"I have $100."
# Code for multiline strings
Downloads.download("https://raw.githubusercontent.com/\
sidooms/MovieTweetings/\
44c525d0c766944910686c60697203cda39305d6/\
snapshots/10K/movies.dat",
"movies.dat")
"a\
b\
c"
# Code for raw strings
"C:\my_folder\my_file.txt"
raw"C:\my_folder\my_file.txt"
# Code for listing 4.2
movies = readlines("movies.dat")
# Code for section 4.2
movie1 = first(movies)
movie1_parts = split(movie1, "::")
supertype(String)
supertype(SubString{String})
# Code for section 4.3
movie1_parts[2]
rx = r"(.*) \((\d{4})\)$"
m = match(rx, movie1_parts[2])
m[1]
m[2]
parse(Int, m[2])
# Code for listing 4.3
function parseline(line::String)
parts = split(line, "::")
m = match(r"(.*) \((\d{4})\)", parts[2])
return (id=parts[1],
name=m[1],
year=parse(Int, m[2]),
genres=split(parts[3], "|"))
end
# Code for parsing one line of movies data
record1 = parseline(movie1)
# Code for listing 4.4
codeunits("a")
codeunits("ε")
codeunits("∀")
# Codes for different patterns of string subsetting
word = first(record1.name, 8)
record1.name[1:8]
for i in eachindex(word)
println(i, ": ", word[i])
end
codeunits("ô")
codeunits("Fantômas")
isascii("Hello world!")
isascii("∀ x: x≥0")
word[1]
word[5]
# Code for section 4.5
records = parseline.(movies)
genres = String[]
for record in records
append!(genres, record.genres)
end
genres
using FreqTables
table = freqtable(genres)
sort!(table)
years = [record.year for record in records]
has_drama = ["Drama" in record.genres for record in records]
drama_prop = proptable(years, has_drama; margins=1)
# Code for listing 4.5
using Plots
plot(names(drama_prop, 1), drama_prop[:, 2]; legend=false,
xlabel="year", ylabel="Drama probability")
# Code for section 4.6.1
s1 = Symbol("x")
s2 = Symbol("hello world!")
s3 = Symbol("x", 1)
typeof(s1)
typeof(s2)
typeof(s3)
Symbol("1")
:x
:x1
:hello world
:1
# Code for section 4.6.2
supertype(Symbol)
:x == :x
:x == :y
# Code for listing 4.6
using BenchmarkTools
str = string.("x", 1:10^6)
symb = Symbol.(str)
@benchmark "x" in $str
@benchmark :x in $symb
# Code for section 4.7
using InlineStrings
s1 = InlineString("x")
typeof(s1)
s2 = InlineString("∀")
typeof(s2)
sv = inlinestrings(["The", "quick", "brown", "fox", "jumps",
"over", "the", "lazy", "dog"])
# Code for listing 4.7
using Random
using BenchmarkTools
Random.seed!(1234);
s1 = [randstring(3) for i in 1:10^6]
s2 = inlinestrings(s1)
# Code for analyzing properties of InlineStrings.jl
Base.summarysize(s1)
Base.summarysize(s2)
@benchmark sort($s1)
@benchmark sort($s2)
# Code for listing 4.8
open("iris.txt", "w") do io
for i in 1:10^6
println(io, "Iris setosa")
println(io, "Iris virginica")
println(io, "Iris versicolor")
end
end
# Code for section 4.8.2
uncompressed = readlines("iris.txt")
using PooledArrays
compressed = PooledArray(uncompressed)
Base.summarysize(uncompressed)
Base.summarysize(compressed)
# Code for section 4.8.3
compressed.invpool
compressed.pool
compressed[10]
compressed.pool[compressed.refs[10]]
Base.summarysize.(compressed.pool)
v1 = string.("x", 1:10^6)
v2 = PooledArray(v1)
Base.summarysize(v1)
Base.summarysize(v2)

359
ch045.jl Normal file
View File

@ -0,0 +1,359 @@
# Bogumił Kamiński, 2021
# Codes for chapter 3
# Code for listing 3.1
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
# Code for checking size of a matrix
size(aq)
size(aq, 1)
size(aq, 2)
# Code comparing tuple to a vector
v = [1, 2, 3]
t = (1, 2, 3)
v[1]
t[1]
v[1] = 10
v
t[1] = 10
# Code for figure 3.2
using BenchmarkTools
@benchmark (1, 2, 3)
@benchmark [1, 2, 3]
# Code for section 3.1.2
using Statistics
mean(aq; dims=1)
std(aq; dims=1)
map(mean, eachcol(aq))
map(std, eachcol(aq))
map(eachcol(aq)) do col
mean(col)
end
[mean(col) for col in eachcol(aq)]
[std(col) for col in eachcol(aq)]
# Code for section 3.1.3
[mean(aq[:, j]) for j in axes(aq, 2)]
[std(aq[:, j]) for j in axes(aq, 2)]
axes(aq, 2)
?Base.OneTo
[mean(view(aq, :, j)) for j in axes(aq, 2)]
[std(@view aq[:, j]) for j in axes(aq, 2)]
# Code for section 3.1.4
using BenchmarkTools
x = ones(10^7, 10)
@benchmark [mean(@view $x[:, j]) for j in axes($x, 2)]
@benchmark [mean($x[:, j]) for j in axes($x, 2)]
@benchmark mean($x, dims=1)
# Code for section 3.1.5
[cor(aq[:, i], aq[:, i+1]) for i in 1:2:7]
collect(1:2:7)
# Code for section 3.1.6
y = aq[:, 2]
X = [ones(11) aq[:, 1]]
X \ y
[[ones(11) aq[:, i]] \ aq[:, i+1] for i in 1:2:7]
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
error = y - prediction
SS_res = sum(v -> v ^ 2, error)
mean_y = mean(y)
SS_tot = sum(v -> (v - mean_y) ^ 2, y)
return 1 - SS_res / SS_tot
end
[R²(aq[:, i], aq[:, i+1]) for i in 1:2:7]
?²
# Code for section 3.1.7
using Plots
scatter(aq[:, 1], aq[:, 2]; legend=false)
plot(scatter(aq[:, 1], aq[:, 2]; legend=false),
scatter(aq[:, 3], aq[:, 4]; legend=false),
scatter(aq[:, 5], aq[:, 6]; legend=false),
scatter(aq[:, 7], aq[:, 8]; legend=false))
plot([scatter(aq[:, i], aq[:, i+1]; legend=false)
for i in 1:2:7]...)
# Code for section 3.2
two_standard = Dict{Int, Int}()
for i in [1, 2, 3, 4, 5, 6]
for j in [1, 2, 3, 4, 5, 6]
s = i + j
if haskey(two_standard, s)
two_standard[s] += 1
else
two_standard[s] = 1
end
end
end
two_standard
keys(two_standard)
values(two_standard)
using Plots
scatter(collect(keys(two_standard)), collect(values(two_standard));
legend=false, xaxis=2:12)
all_dice = [[1, x2, x3, x4, x5, x6]
for x2 in 2:11
for x3 in x2:11
for x4 in x3:11
for x5 in x4:11
for x6 in x5:11]
for d1 in all_dice, d2 in all_dice
test = Dict{Int, Int}()
for i in d1, j in d2
s = i + j
if haskey(test, s)
test[s] += 1
else
test[s] = 1
end
end
if test == two_standard
println(d1, " ", d2)
end
end
# Code for section 3.3
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
dataset1 = (x=aq[:, 1], y=aq[:, 2])
dataset1[1]
dataset1.x
# Code for listing 3.2
data = (set1=(x=aq[:, 1], y=aq[:, 2]),
set2=(x=aq[:, 3], y=aq[:, 4]),
set3=(x=aq[:, 5], y=aq[:, 6]),
set4=(x=aq[:, 7], y=aq[:, 8]))
# Code for section 3.3.2
using Statistics
map(s -> mean(s.x), data)
map(s -> cor(s.x, s.y), data)
using GLM
model = lm(@formula(y ~ x), data.set1)
r2(model)
# Code for section 3.3.3
model.mm
x = [3, 1, 2]
sort(x)
x
sort!(x)
x
empty_field!(nt, i) = empty!(nt[i])
nt = (dict = Dict("a" => 1, "b" => 2), int=10)
empty_field!(nt, 1)
nt
# Code for section 3.4.1
x = [1 2 3]
y = [1, 2, 3]
x * y
a = [1, 2, 3]
b = [4, 5, 6]
a * b
a .* b
map(*, a, b)
[a[i] * b[i] for i in eachindex(a, b)]
eachindex(a, b)
eachindex([1, 2, 3], [4, 5])
map(*, [1, 2, 3], [4, 5])
[1, 2, 3] .* [4, 5]
# Code for section 3.4.2
[1, 2, 3] .* [4]
[1, 2, 3] .^ 2
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10] .* [1 2 3 4 5 6 7 8 9 10]
["x", "y", "z"] .=> [sum minimum maximum]
abs.([1, -2, 3, -4])
abs([1, 2, 3])
string(1, 2, 3)
string.("x", 1:10)
f(i::Int) = string("got integer ", i)
f(s::String) = string("got string ", s)
f.([1, "1"])
# Code for section 3.4.3
in(1, [1, 2, 3])
in(4, [1, 2, 3])
in([1, 3, 5, 7, 9], [1, 2, 3, 4])
in.([1, 3, 5, 7, 9], [1, 2, 3, 4])
in.([1, 3, 5, 7, 9], Ref([1, 2, 3, 4]))
# Code for section 3.4.4
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89]
using Statistics
mean.(eachcol(aq))
mean(eachcol(aq))
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
error = y - prediction
SS_res = sum(v -> v ^ 2, error)
mean_y = mean(y)
SS_tot = sum(v -> (v - mean_y) ^ 2, y)
return 1 - SS_res / SS_tot
end
function R²(x, y)
X = [ones(11) x]
model = X \ y
prediction = X * model
SS_res = sum((y .- prediction) .^ 2)
SS_tot = sum((y .- mean(y)) .^ 2)
return 1 - SS_res / SS_tot
end
# Code for section 3.5
[]
Dict()
Float64[1, 2, 3]
Dict{UInt8, Float64}(0 => 0, 1 => 1)
UInt32(200)
Real[1, 1.0, 0x3]
v1 = Any[1, 2, 3]
eltype(v1)
v2 = Float64[1, 2, 3]
eltype(v2)
v3 = [1, 2, 3]
eltype(v2)
d1 = Dict()
eltype(d1)
d2 = Dict(1 => 2, 3 => 4)
eltype(d2)
p = 1 => 2
typeof(p)
# Code for section 3.5.1
[1, 2, 3] isa AbstractVector{Int}
[1, 2, 3] isa AbstractVector{Real}
AbstractVector{<:Real}
# Code for section 3.5.2
using Statistics
function ourcov(x::AbstractVector{<:Real},
y::AbstractVector{<:Real})
len = length(x)
@assert len == length(y) > 0
return sum((x .- mean(x)) .* (y .- mean(y))) / (len - 1)
end
ourcov(1:4, [1.0, 3.0, 2.0, 4.0])
cov(1:4, [1.0, 3.0, 2.0, 4.0])
ourcov(1:4, Any[1.0, 3.0, 2.0, 4.0])
x = Any[1, 2, 3]
identity.(x)
y = Any[1, 2.0]
identity.(y)

214
ch05.jl
View File

@ -1,214 +0,0 @@
# Bogumił Kamiński, 2022
# Codes for chapter 5
# Code for listing 5.1
using HTTP
using JSON3
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-01/?format=json"
response = HTTP.get(query)
json = JSON3.read(response.body)
# Code for the remainder of section 5.1.2
response.body
String(response.body)
response.body
json.table
json.currency
json.code
json.rates
json.rates[1].mid
only(json.rates).mid
only([])
only([1, 2])
# Code for listing 5.2
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-06/?format=json"
response = HTTP.get(query)
# Code for listing 5.3
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-01/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
missing
else
rethrow(e)
end
end
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-06/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
missing
else
rethrow(e)
end
end
# Code for section 5.2
ismissing(missing)
ismissing(1)
1 + missing
sin(missing)
1 == missing
1 > missing
1 < missing
if missing
print("this is not printed")
end
missing && true
coalesce(missing, true)
coalesce(missing, false)
isequal(1, missing)
isequal(missing, missing)
isless(1, missing)
isless(missing, missing)
isless(Inf, missing)
a = [1]
b = [1]
isequal(a, b)
a === b
x = [1, missing, 3, 4, missing]
coalesce.(x, 0)
sum(x)
y = skipmissing(x)
sum(y)
sum(skipmissing(x))
fun(x::Int, y::Int) = x + y
fun(1, 2)
fun(1, missing)
using Missings
fun2 = passmissing(fun)
fun2(1, 2)
fun2(1, missing)
# Code for section 5.3
using Dates
d = Date("2020-06-01")
typeof(d)
year(d)
month(d)
day(d)
dayofweek(d)
dayname(d)
Date(2020, 6, 1)
dates = Date.(2020, 6, 1:30)
Day(1)
d
d + Day(1)
Date(2020, 5, 20):Day(1):Date(2020, 7, 5)
collect(Date(2020, 5, 20):Day(1):Date(2020, 7, 5))
# Code for listing 5.6
function get_rate(date::Date)
query = "https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$date/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
return only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
return missing
else
rethrow(e)
end
end
end
# Code for showing how string interpolation works
"https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$(dates[1])/?format=json"
"https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$dates[1]/?format=json"
# Code for listing 5.7
rates = get_rate.(dates)
# Code for section 5.4
using Statistics
mean(rates)
std(rates)
mean(skipmissing(rates))
std(skipmissing(rates))
# Code for listing 5.8
using FreqTables
proptable(dayname.(dates), ismissing.(rates); margins=1)
# Code showing how to specify a complex condition using broadcasting
dayname.(dates) .== "Thursday" .&& ismissing.(rates)
# Code for listing 5.9
dates[dayname.(dates) .== "Thursday" .&& ismissing.(rates)]
# Codes for plotting exchange rate data
using Plots
plot(dates, rates; xlabel="day", ylabel="PLN/USD", legend=false)
rates_ok = .!ismissing.(rates)
plot(dates[rates_ok], rates[rates_ok];
xlabel="day", ylabel="PLN/USD", legend=false)
using Impute
rates_filled = Impute.interp(rates)
scatter!(dates, rates_filled)

370
ch06.jl
View File

@ -1,248 +1,224 @@
# Bogumił Kamiński, 2022
# Codes for chapter 6
# Codes for chapter 4
# Code for section 6.1
# Code for listing 4.1
if isfile("puzzles.csv.bz2")
@info "file already present"
else
@info "fetching file"
download("https://database.lichess.org/" *
"lichess_db_puzzle.csv.bz2",
"puzzles.csv.bz2")
import Downloads
Downloads.download("https://raw.githubusercontent.com/" *
"sidooms/MovieTweetings/" *
"44c525d0c766944910686c60697203cda39305d6/" *
"snapshots/10K/movies.dat",
"movies.dat")
# Code for string interpolation examples
x = 10
"I have $x apples"
"I have \$100."
"I have $100."
# Code for multiline strings
Downloads.download("https://raw.githubusercontent.com/\
sidooms/MovieTweetings/\
44c525d0c766944910686c60697203cda39305d6/\
snapshots/10K/movies.dat",
"movies.dat")
"a\
b\
c"
# Code for raw strings
"C:\my_folder\my_file.txt"
raw"C:\my_folder\my_file.txt"
# Code for listing 4.2
movies = readlines("movies.dat")
# Code for section 4.2
movie1 = first(movies)
movie1_parts = split(movie1, "::")
supertype(String)
supertype(SubString{String})
# Code for section 4.3
movie1_parts[2]
rx = r"(.*) \((\d{4})\)$"
m = match(rx, movie1_parts[2])
m[1]
m[2]
parse(Int, m[2])
# Code for listing 4.3
function parseline(line::String)
parts = split(line, "::")
m = match(r"(.*) \((\d{4})\)", parts[2])
return (id=parts[1],
name=m[1],
year=parse(Int, m[2]),
genres=split(parts[3], "|"))
end
using CodecBzip2
compressed = read("puzzles.csv.bz2")
plain = transcode(Bzip2Decompressor, compressed)
# Code for parsing one line of movies data
open("puzzles.csv", "w") do io
println(io, "PuzzleId,FEN,Moves,Rating,RatingDeviation," *
"Popularity,NbPlays,Themes,GameUrl")
write(io, plain)
record1 = parseline(movie1)
# Code for listing 4.4
codeunits("a")
codeunits("ε")
codeunits("∀")
# Codes for different patterns of string subsetting
word = first(record1.name, 8)
record1.name[1:8]
for i in eachindex(word)
println(i, ": ", word[i])
end
readlines("puzzles.csv")
codeunits("ô")
# Code for section 6.2
codeunits("Fantômas")
using CSV
using DataFrames
puzzles = CSV.read("puzzles.csv", DataFrame);
isascii("Hello world!")
isascii("∀ x: x≥0")
CSV.read(plain, DataFrame);
word[1]
word[5]
compressed = nothing
plain = nothing
# Code for section 4.5
# Code for listing 6.1
records = parseline.(movies)
puzzles
genres = String[]
for record in records
append!(genres, record.genres)
end
genres
# Code for listing 6.2
using FreqTables
table = freqtable(genres)
sort!(table)
describe(puzzles)
years = [record.year for record in records]
has_drama = ["Drama" in record.genres for record in records]
drama_prop = proptable(years, has_drama; margins=1)
# Code for getting basic information about a data frame
ncol(puzzles)
nrow(puzzles)
names(puzzles)
# Code for section 6.3
puzzles.Rating
using BenchmarkTools
@benchmark $puzzles.Rating
puzzles.Rating == copy(puzzles.Rating)
puzzles.Rating === copy(puzzles.Rating)
puzzles.Rating === puzzles.Rating
copy(puzzles.Rating) === copy(puzzles.Rating)
puzzles."Rating"
col = "Rating"
data_frame_name[selected_rows, selected_columns]
puzzles[:, "Rating"]
puzzles[:, :Rating]
puzzles[:, 4]
puzzles[:, col]
columnindex(puzzles, "Rating")
columnindex(puzzles, "Some fancy column name")
hasproperty(puzzles, "Rating")
hasproperty(puzzles, "Some fancy column name")
@benchmark $puzzles[:, :Rating]
puzzles[!, "Rating"]
puzzles[!, :Rating]
puzzles[!, 4]
puzzles[!, col]
# Code for listing 4.5
using Plots
plot(histogram(puzzles.Rating, label="Rating"),
histogram(puzzles.RatingDeviation, label="RatingDeviation"),
histogram(puzzles.Popularity, label="Popularity"),
histogram(puzzles.NbPlays, label="NbPlays"))
plot([histogram(puzzles[!, col]; label=col) for
col in ["Rating", "RatingDeviation",
"Popularity", "NbPlays"]]...)
plot(names(drama_prop, 1), drama_prop[:, 2]; legend=false,
xlabel="year", ylabel="Drama probability")
# Code for section 6.4
# Code for section 4.6.1
using Statistics
plays_lo = median(puzzles.NbPlays)
puzzles.NbPlays .> plays_lo
s1 = Symbol("x")
s2 = Symbol("hello world!")
s3 = Symbol("x", 1)
puzzles.NbPlays > plays_lo
typeof(s1)
typeof(s2)
typeof(s3)
rating_lo = 1500
rating_hi = quantile(puzzles.Rating, 0.99)
rating_lo .< puzzles.Rating .< rating_hi
Symbol("1")
row_selector = (puzzles.NbPlays .> plays_lo) .&&
(rating_lo .< puzzles.Rating .< rating_hi)
:x
:x1
sum(row_selector)
count(row_selector)
:hello world
:1
# Code for listing 6.3
# Code for section 4.6.2
good = puzzles[row_selector, ["Rating", "Popularity"]]
supertype(Symbol)
# Code for plotting histograms
:x == :x
:x == :y
plot(histogram(good.Rating; label="Rating"),
histogram(good.Popularity; label="Popularity"))
# Code for listing 4.6
# Code for column selectors
using BenchmarkTools
str = string.("x", 1:10^6)
symb = Symbol.(str)
@benchmark "x" in $str
@benchmark :x in $symb
puzzles[1, "Rating"]
# Code for section 4.7
puzzles[:, "Rating"]
using InlineStrings
s1 = InlineString("x")
typeof(s1)
s2 = InlineString("∀")
typeof(s2)
sv = inlinestrings(["The", "quick", "brown", "fox", "jumps",
"over", "the", "lazy", "dog"])
row1 = puzzles[1, ["Rating", "Popularity"]]
# Code for listing 4.7
row1["Rating"]
row1[:Rating]
row1[1]
row1.Rating
row1."Rating"
using Random
using BenchmarkTools
Random.seed!(1234);
s1 = [randstring(3) for i in 1:10^6]
s2 = inlinestrings(s1)
good = puzzles[row_selector, ["Rating", "Popularity"]]
# Code for analyzing properties of InlineStrings.jl
good[1, "Rating"]
good[1, :]
good[:, "Rating"]
good[:, :]
Base.summarysize(s1)
Base.summarysize(s2)
names(puzzles, ["Rating", "Popularity"])
names(puzzles, [:Rating, :Popularity])
names(puzzles, [4, 6])
names(puzzles, [false, false, false, true, false, true, false, false, false])
names(puzzles, r"Rating")
names(puzzles, Not([4, 6]))
names(puzzles, Not(r"Rating"))
names(puzzles, Between("Rating", "Popularity"))
names(puzzles, :)
names(puzzles, All())
names(puzzles, Cols(r"Rating", "NbPlays"))
names(puzzles, Cols(startswith("P")))
@benchmark sort($s1)
@benchmark sort($s2)
names(puzzles, startswith("P"))
# Code for listing 4.8
names(puzzles, Real)
names(puzzles, AbstractString)
puzzles[:, names(puzzles, Real)]
# Code for row subsetting
df1 = puzzles[:, ["Rating", "Popularity"]];
df2 = puzzles[!, ["Rating", "Popularity"]];
df1 == df2
df1 == puzzles
df2 == puzzles
df1.Rating === puzzles.Rating
df1.Popularity === puzzles.Popularity
df2.Rating === puzzles.Rating
df2.Popularity === puzzles.Popularity
@benchmark $puzzles[:, ["Rating", "Popularity"]]
@benchmark $puzzles[!, ["Rating", "Popularity"]]
puzzles[1, 1]
puzzles[[1], 1]
puzzles[1, [1]]
puzzles[[1], [1]]
# Code for making views
@view puzzles[1, 1]
@view puzzles[[1], 1]
@view puzzles[1, [1]]
@view puzzles[[1], [1]]
@btime $puzzles[$row_selector, ["Rating", "Popularity"]];
@btime @view $puzzles[$row_selector, ["Rating", "Popularity"]];
parentindices(@view puzzles[row_selector, ["Rating", "Popularity"]])
# Code for section 6.5
describe(good)
rating_mapping = Dict{Int, Vector{Int}}()
for (i, rating) in enumerate(good.Rating)
if haskey(rating_mapping, rating)
push!(rating_mapping[rating], i)
else
rating_mapping[rating] = [i]
open("iris.txt", "w") do io
for i in 1:10^6
println(io, "Iris setosa")
println(io, "Iris virginica")
println(io, "Iris versicolor")
end
end
rating_mapping
good[rating_mapping[2108], :]
# Code for section 4.8.2
unique(good[rating_mapping[2108], :].Rating)
uncompressed = readlines("iris.txt")
using Statistics
mean(good[rating_mapping[2108], "Popularity"])
using PooledArrays
compressed = PooledArray(uncompressed)
ratings = unique(good.Rating)
Base.summarysize(uncompressed)
Base.summarysize(compressed)
mean_popularities = map(ratings) do rating
indices = rating_mapping[rating]
popularities = good[indices, "Popularity"]
return mean(popularities)
end
# Code for section 4.8.3
scatter(ratings, mean_popularities;
xlabel="rating", ylabel="mean popularity", legend=false)
compressed.invpool
compressed.pool
import Loess
model = Loess.loess(ratings, mean_popularities);
ratings_predict = float.(sort(ratings))
popularity_predict = Loess.predict(model, ratings_predict)
compressed[10]
compressed.pool[compressed.refs[10]]
plot!(ratings_predict, popularity_predict; width=5, color="black")
Base.summarysize.(compressed.pool)
v1 = string.("x", 1:10^6)
v2 = PooledArray(v1)
Base.summarysize(v1)
Base.summarysize(v2)

431
ch07.jl
View File

@ -1,279 +1,214 @@
# Bogumił Kamiński, 2022
# Codes for chapter 7
# Codes for chapter 5
# Code for section 7.1
# Code for listing 5.1
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89];
using HTTP
using JSON3
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-01/?format=json"
response = HTTP.get(query)
json = JSON3.read(response.body)
data = (set1=(x=aq[:, 1], y=aq[:, 2]),
set2=(x=aq[:, 3], y=aq[:, 4]),
set3=(x=aq[:, 5], y=aq[:, 6]),
set4=(x=aq[:, 7], y=aq[:, 8]));
# Code for the remainder of section 5.1.2
using DataFrames
response.body
# Code for listing 7.1
String(response.body)
aq1 = ataFrame(aq, ["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4"])
DataFrame(aq, [:x1, :y1, :x2, :y2, :x3, :y3, :x4, :y4])
response.body
# Code for creating DataFrame with automatic column names
json.table
json.currency
json.code
json.rates
DataFrame(aq, :auto)
json.rates[1].mid
# Codes for creating DataFrame from vector of vectors
only(json.rates).mid
aq_vec = collect(eachcol(aq))
DataFrame(aq_vec, ["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4"])
DataFrame(aq_vec, :auto)
only([])
only([1, 2])
# Codes for section 7.1.2
# Code for listing 5.2
data.set1.x
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-06/?format=json"
response = HTTP.get(query)
DataFrame(x1=data.set1.x, y1=data.set1.y,
x2=data.set2.x, y2=data.set2.y,
x3=data.set3.x, y3=data.set3.y,
x4=data.set4.x, y4=data.set4.y)
# Code for listing 5.3
DataFrame(:x1 => data.set1.x, :y1 => data.set1.y,
:x2 => data.set2.x, :y2 => data.set2.y,
:x3 => data.set3.x, :y3 => data.set3.y,
:x4 => data.set4.x, :y4 => data.set4.y)
DataFrame([:x1 => data.set1.x, :y1 => data.set1.y,
:x2 => data.set2.x, :y2 => data.set2.y,
:x3 => data.set3.x, :y3 => data.set3.y,
:x4 => data.set4.x, :y4 => data.set4.y]);
[(i, v) for i in 1:4 for v in [:x, :y]]
[string(v, i) for i in 1:4 for v in [:x, :y]]
[string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]]
DataFrame([string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]]);
data_dict = Dict([string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]])
collect(data_dict)
DataFrame(data_dict)
df1 = DataFrame(x1=data.set1.x)
df1.x1 === data.set1.x
df2 = DataFrame(x1=data.set1.x; copycols=false)
df2.x1 === data.set1.x
df = DataFrame(x=1:3, y=1)
df.x
DataFrame(x=[1], y=[1, 2, 3])
# Codes for section 7.1.3
data.set1
DataFrame(data.set1)
DataFrame([(a=1, b=2), (a=3, b=4), (a=5, b=6)])
data
# Code for listing 7.2
aq2 = DataFrame(data)
# Codes for listing 7.3
data_dfs = map(DataFrame, data)
# Codes for vertical concatenation examples
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4)
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4;
source="source_id")
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4;
source="source_id"=>string.("set", 1:4))
reduce(vcat, collect(data_dfs);
source="source_id"=>string.("set", 1:4))
# Code for listing 7.4
df1 = DataFrame(a=1:3, b=11:13)
df2 = DataFrame(a=4:6, c=24:26)
vcat(df1, df2)
vcat(df1, df2; cols=:union)
# Code for listing 7.5
df_agg = DataFrame()
append!(df_agg, data_dfs.set1)
append!(df_agg, data_dfs.set2)
# Code for appending tables to a data frame
df_agg = DataFrame()
append!(df_agg, data.set1)
append!(df_agg, data.set2)
# Code for promote keyword argument
df1 = DataFrame(a=1:3, b=11:13)
df2 = DataFrame(a=4:6, b=[14, missing, 16])
append!(df1, df2)
append!(df1, df2; promote=true)
# Code for section 7.2.3
df = DataFrame()
push!(df, (a=1, b=2))
push!(df, (a=3, b=4))
df = DataFrame(a=Int[], b=Int[])
push!(df, [1, 2])
push!(df, [3, 4])
function sim_step(current)
dx, dy = rand(((1,0), (-1,0), (0,1), (0,-1)))
return (x=current.x + dx, y=current.y + dy)
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-01/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
missing
else
rethrow(e)
end
end
using BenchmarkTools
@btime rand(((1,0), (-1,0), (0,1), (0,-1)));
query = "https://api.nbp.pl/api/exchangerates/rates/a/usd/" *
"2020-06-06/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
missing
else
rethrow(e)
end
end
dx, dy = (10, 20)
dx
dy
# Code for section 5.2
ismissing(missing)
ismissing(1)
1 + missing
sin(missing)
1 == missing
1 > missing
1 < missing
if missing
print("this is not printed")
end
missing && true
coalesce(missing, true)
coalesce(missing, false)
isequal(1, missing)
isequal(missing, missing)
isless(1, missing)
isless(missing, missing)
isless(Inf, missing)
a = [1]
b = [1]
isequal(a, b)
a === b
x = [1, missing, 3, 4, missing]
coalesce.(x, 0)
sum(x)
y = skipmissing(x)
sum(y)
sum(skipmissing(x))
fun(x::Int, y::Int) = x + y
fun(1, 2)
fun(1, missing)
using Missings
fun2 = passmissing(fun)
fun2(1, 2)
fun2(1, missing)
# Code for section 5.3
using Dates
d = Date("2020-06-01")
typeof(d)
year(d)
month(d)
day(d)
dayofweek(d)
dayname(d)
Date(2020, 6, 1)
dates = Date.(2020, 6, 1:30)
Day(1)
d
d + Day(1)
Date(2020, 5, 20):Day(1):Date(2020, 7, 5)
collect(Date(2020, 5, 20):Day(1):Date(2020, 7, 5))
# Code for listing 5.6
function get_rate(date::Date)
query = "https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$date/?format=json"
try
response = HTTP.get(query)
json = JSON3.read(response.body)
return only(json.rates).mid
catch e
if e isa HTTP.ExceptionRequest.StatusError
return missing
else
rethrow(e)
end
end
end
# Code for showing how string interpolation works
"https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$(dates[1])/?format=json"
"https://api.nbp.pl/api/exchangerates/rates/" *
"a/usd/$dates[1]/?format=json"
# Code for listing 5.7
rates = get_rate.(dates)
# Code for section 5.4
using Statistics
mean(rates)
std(rates)
mean(skipmissing(rates))
std(skipmissing(rates))
# Code for listing 5.8
using FreqTables
using Random
Random.seed!(1234);
proptable([rand(((1,0), (-1,0), (0,1), (0,-1))) for _ in 1:10^7])
proptable(dayname.(dates), ismissing.(rates); margins=1)
using Random
Random.seed!(6);
walk = DataFrame(x=0, y=0)
for _ in 1:10
current = walk[end, :]
push!(walk, sim_step(current))
end
walk
# Code showing how to specify a complex condition using broadcasting
plot(walk.x, walk.y;
legend=false,
series_annotations=1:11,
xticks=range(extrema(walk.x)...),
yticks=range(extrema(walk.y)...))
dayname.(dates) .== "Thursday" .&& ismissing.(rates)
extrema(walk.y)
# Code for listing 5.9
range(1, 5)
dates[dayname.(dates) .== "Thursday" .&& ismissing.(rates)]
(3/4)^9
# Codes for plotting exchange rate data
# Code for listing 7.6
using Plots
plot(dates, rates; xlabel="day", ylabel="PLN/USD", legend=false)
function walk_unique() #A
walk = DataFrame(x=0, y=0)
for _ in 1:10
current = walk[end, :]
push!(walk, sim_step(current))
end
return nrow(unique(walk)) == nrow(walk) #B
end
Random.seed!(2);
proptable([walk_unique() for _ in 1:10^5])
rates_ok = .!ismissing.(rates)
# Code for a note on conversion
plot(dates[rates_ok], rates[rates_ok];
xlabel="day", ylabel="PLN/USD", legend=false)
x = [1.5]
x[1] = 1
x
using Impute
rates_filled = Impute.interp(rates)
# Code from section 7.3.1
Matrix(walk)
Matrix{Any}(walk)
Matrix{String}(walk)
plot(walk)
plot(Matrix(walk); labels=["x" "y"] , legend=:topleft)
# Code from section 7.3.2
Tables.columntable(walk)
using BenchmarkTools
function mysum(table)
s = 0 #A
for v in table.x #B
s += v
end
return s
end
df = DataFrame(x=1:1_000_000);
@btime mysum($df)
tab = Tables.columntable(df);
@btime mysum($tab)
@code_warntype mysum(df)
@code_warntype mysum(tab)
typeof(tab)
function barrier_mysum2(x)
s = 0
for v in x
s += v
end
return s
end
mysum2(table) = barrier_mysum2(table.x)
@btime mysum2($df)
df = DataFrame(a=[1, 1, 2], b=[1, 1, 2])
unique(df)
tab = Tables.columntable(df)
unique(tab)
# Code from section 7.3.3
Tables.rowtable(walk)
nti = Tables.namedtupleiterator(walk)
for v in nti
println(v)
end
er = eachrow(walk)
er[1]
er[end]
ec = eachcol(walk)
ec[1]
ec[end]
identity.(eachcol(walk))
df = DataFrame(x=1:2, b=["a", "b"])
identity.(eachcol(df))
scatter!(dates, rates_filled)

420
ch08.jl
View File

@ -1,284 +1,248 @@
# Bogumił Kamiński, 2022
# Codes for chapter 8
# Codes for chapter 6
# Codes for section 8.1
# Code for section 6.1
# Code for listing 8.1
import Downloads
using SHA
git_zip = "git_web_ml.zip"
if !isfile(git_zip)
Downloads.download("https://snap.stanford.edu/data/" *
"git_web_ml.zip",
git_zip)
end
isfile(git_zip)
open(sha256, git_zip) == [0x56, 0xc0, 0xc1, 0xc2,
0xc4, 0x60, 0xdc, 0x4c,
0x7b, 0xf8, 0x93, 0x57,
0xb1, 0xfe, 0xc0, 0x20,
0xf4, 0x5e, 0x2e, 0xce,
0xba, 0xb8, 0x1d, 0x13,
0x1d, 0x07, 0x3b, 0x10,
0xe2, 0x8e, 0xc0, 0x31]
# Code for opeining a zip archive
import ZipFile
git_archive = ZipFile.Reader(git_zip)
# Code for listing 8.2
function ingest_to_df(archive::ZipFile.Reader, filename::AbstractString)
idx = only(findall(x -> x.name == filename, archive.files))
return CSV.read(read(archive.files[idx]), DataFrame)
if isfile("puzzles.csv.bz2")
@info "file already present"
else
@info "fetching file"
download("https://database.lichess.org/" *
"lichess_db_puzzle.csv.bz2",
"puzzles.csv.bz2")
end
# Code for working with zip archive
using CodecBzip2
compressed = read("puzzles.csv.bz2")
plain = transcode(Bzip2Decompressor, compressed)
git_archive.files
open("puzzles.csv", "w") do io
println(io, "PuzzleId,FEN,Moves,Rating,RatingDeviation," *
"Popularity,NbPlays,Themes,GameUrl")
write(io, plain)
end
git_archive.files[2].name
readlines("puzzles.csv")
findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files)
findall(x -> x.name == "", git_archive.files)
only(findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files))
only(findall(x -> x.name == "", git_archive.files))
# Code for listing 8.3
# Code for section 6.2
using CSV
using DataFrames
edges_df = ingest_to_df(git_archive, "git_web_ml/musae_git_edges.csv");
classes_df = ingest_to_df(git_archive, "git_web_ml/musae_git_target.csv");
close(git_archive)
summary(edges_df)
describe(edges_df, :min, :max, :mean, :nmissing, :eltype)
summary(classes_df)
describe(classes_df, :min, :max, :mean, :nmissing, :eltype)
puzzles = CSV.read("puzzles.csv", DataFrame);
# Code for updating data frame columns using broadcasting
CSV.read(plain, DataFrame);
edges_df .+= 1
classes_df.id .+= 1
compressed = nothing
plain = nothing
# Code for examples of data frame broadcasting
# Code for listing 6.1
df = DataFrame(a=1:3, b=[4, missing, 5])
df .^ 2
coalesce.(df, 0)
df .+ [10, 11, 12]
puzzles
# Code for checking the order of :id column in a data frame
# Code for listing 6.2
classes_df.id == axes(classes_df, 1)
describe(puzzles)
# Code for the difference between ! and : in broadcasting assignment
# Code for getting basic information about a data frame
df = DataFrame(a=1:3, b=1:3)
df[!, :a] .= "x"
df[:, :b] .= "x"
df
ncol(puzzles)
# Code for the difference between ! and : in assignment
nrow(puzzles)
df = DataFrame(a=1:3, b=1:3, c=1:3)
df[!, :a] = ["x", "y", "z"]
df[:, :b] = ["x", "y", "z"]
df[:, :c] = [11, 12, 13]
df
names(puzzles)
# Codes for section 8.2
# Code for section 6.3
# Code from listing 8.4
puzzles.Rating
using Graphs
gh = SimpleGraph(nrow(classes_df))
for (from, to) in eachrow(edges_df)
add_edge!(gh, from, to)
end
gh
ne(gh)
nv(gh)
using BenchmarkTools
@benchmark $puzzles.Rating
# Code for iterator destruction in iteration specification
puzzles.Rating == copy(puzzles.Rating)
mat = [1 2; 3 4; 5 6]
for (x1, x2) in eachrow(mat)
@show x1, x2
end
puzzles.Rating === copy(puzzles.Rating)
# Code for getting degrees of nodes in the graph
puzzles.Rating === puzzles.Rating
degree(gh)
copy(puzzles.Rating) === copy(puzzles.Rating)
# Code for adding a column to a data frame
puzzles."Rating"
classes_df.deg = degree(gh)
col = "Rating"
# Code for the difference between ! and : when adding a column
data_frame_name[selected_rows, selected_columns]
df = DataFrame()
x = [1, 2, 3]
df[!, :x1] = x
df[:, :x2] = x
df
df.x1 === x
df.x2 === x
df.x2 == x
puzzles[:, "Rating"]
puzzles[:, :Rating]
puzzles[:, 4]
puzzles[:, col]
# Code for creating a column using broadcasting
columnindex(puzzles, "Rating")
df.x3 .= 1
df
columnindex(puzzles, "Some fancy column name")
# Code for edge iterator of a graph
hasproperty(puzzles, "Rating")
hasproperty(puzzles, "Some fancy column name")
edges(gh)
@benchmark $puzzles[:, :Rating]
e1 = first(edges(gh))
dump(e1)
e1.src
e1.dst
# Code for listing 8.5
function deg_class(gh, class)
deg_ml = zeros(Int, length(class))
deg_web = zeros(Int, length(class))
for edge in edges(gh)
a, b = edge.src, edge.dst
if class[b] == 1
deg_ml[a] += 1
else
deg_web[a] += 1
end
if class[a] == 1
deg_ml[b] += 1
else
deg_web[b] += 1
end
end
return (deg_ml, deg_web)
end
# Code for computing machine learning and web neighbors for gh graph
classes_df.deg_ml, classes_df.deg_web =
deg_class(gh, classes_df.ml_target)
# Code for checking type stability of deg_class function
@time deg_class(gh, classes_df.ml_target);
@code_warntype deg_class(gh, classes_df.ml_target)
# Code for checking the classes_df summary statistics
describe(classes_df, :min, :max, :mean, :std)
# Code for average degree of node in the graph
2 * ne(gh) / nv(gh)
# Code for checking correctness of computations
classes_df.deg_ml + classes_df.deg_web == classes_df.deg
# Code for showing that DataFrames.jl checks consistency of stored objects
df = DataFrame(a=1, b=11)
push!(df.a, 2)
df
# Codes for section 8.3
# Code for computing groupwise means of columns
using Statistics
for type in [0, 1], col in ["deg_ml", "deg_web"]
println((type, col, mean(classes_df[classes_df.ml_target .== type, col])))
end
gdf = groupby(classes_df, :ml_target)
combine(gdf,
:deg_ml => mean => :mean_deg_ml,
:deg_web => mean => :mean_deg_web)
using DataFramesMeta
@combine(gdf,
:mean_deg_ml = mean(:deg_ml),
:mean_deg_web = mean(:deg_web))
# Code for simple plotting of relationship between developer degree and type
puzzles[!, "Rating"]
puzzles[!, :Rating]
puzzles[!, 4]
puzzles[!, col]
using Plots
scatter(classes_df.deg_ml, classes_df.deg_web;
color=[x == 1 ? "black" : "gray" for x in classes_df.ml_target],
xlabel="degree ml", ylabel="degree web", labels=false)
plot(histogram(puzzles.Rating, label="Rating"),
histogram(puzzles.RatingDeviation, label="RatingDeviation"),
histogram(puzzles.Popularity, label="Popularity"),
histogram(puzzles.NbPlays, label="NbPlays"))
# Code for aggregation of degree data
plot([histogram(puzzles[!, col]; label=col) for
col in ["Rating", "RatingDeviation",
"Popularity", "NbPlays"]]...)
agg_df = combine(groupby(classes_df, [:deg_ml, :deg_web]),
:ml_target => (x -> 1 - mean(x)) => :web_mean)
# Code for section 6.4
# Code for comparison how Julia parses expressions
using Statistics
plays_lo = median(puzzles.NbPlays)
puzzles.NbPlays .> plays_lo
:ml_target => (x -> 1 - mean(x)) => :web_mean
:ml_target => x -> 1 - mean(x) => :web_mean
puzzles.NbPlays > plays_lo
# Code for aggregation using DataFramesMeta.jl
rating_lo = 1500
rating_hi = quantile(puzzles.Rating, 0.99)
rating_lo .< puzzles.Rating .< rating_hi
@combine(groupby(classes_df, [:deg_ml, :deg_web]),
:web_mean = 1 - mean(:ml_target))
row_selector = (puzzles.NbPlays .> plays_lo) .&&
(rating_lo .< puzzles.Rating .< rating_hi)
# Code for getting summary information about the aggregated data frame
sum(row_selector)
count(row_selector)
describe(agg_df)
# Code for listing 6.3
# Code for log1p function
good = puzzles[row_selector, ["Rating", "Popularity"]]
log1p(0)
# Code for plotting histograms
# Code for listing 8.6
plot(histogram(good.Rating; label="Rating"),
histogram(good.Popularity; label="Popularity"))
function gen_ticks(maxv)
max2 = round(Int, log2(maxv))
tick = [0; 2 .^ (0:max2)]
return (log1p.(tick), tick)
# Code for column selectors
puzzles[1, "Rating"]
puzzles[:, "Rating"]
row1 = puzzles[1, ["Rating", "Popularity"]]
row1["Rating"]
row1[:Rating]
row1[1]
row1.Rating
row1."Rating"
good = puzzles[row_selector, ["Rating", "Popularity"]]
good[1, "Rating"]
good[1, :]
good[:, "Rating"]
good[:, :]
names(puzzles, ["Rating", "Popularity"])
names(puzzles, [:Rating, :Popularity])
names(puzzles, [4, 6])
names(puzzles, [false, false, false, true, false, true, false, false, false])
names(puzzles, r"Rating")
names(puzzles, Not([4, 6]))
names(puzzles, Not(r"Rating"))
names(puzzles, Between("Rating", "Popularity"))
names(puzzles, :)
names(puzzles, All())
names(puzzles, Cols(r"Rating", "NbPlays"))
names(puzzles, Cols(startswith("P")))
names(puzzles, startswith("P"))
names(puzzles, Real)
names(puzzles, AbstractString)
puzzles[:, names(puzzles, Real)]
# Code for row subsetting
df1 = puzzles[:, ["Rating", "Popularity"]];
df2 = puzzles[!, ["Rating", "Popularity"]];
df1 == df2
df1 == puzzles
df2 == puzzles
df1.Rating === puzzles.Rating
df1.Popularity === puzzles.Popularity
df2.Rating === puzzles.Rating
df2.Popularity === puzzles.Popularity
@benchmark $puzzles[:, ["Rating", "Popularity"]]
@benchmark $puzzles[!, ["Rating", "Popularity"]]
puzzles[1, 1]
puzzles[[1], 1]
puzzles[1, [1]]
puzzles[[1], [1]]
# Code for making views
@view puzzles[1, 1]
@view puzzles[[1], 1]
@view puzzles[1, [1]]
@view puzzles[[1], [1]]
@btime $puzzles[$row_selector, ["Rating", "Popularity"]];
@btime @view $puzzles[$row_selector, ["Rating", "Popularity"]];
parentindices(@view puzzles[row_selector, ["Rating", "Popularity"]])
# Code for section 6.5
describe(good)
rating_mapping = Dict{Int, Vector{Int}}()
for (i, rating) in enumerate(good.Rating)
if haskey(rating_mapping, rating)
push!(rating_mapping[rating], i)
else
rating_mapping[rating] = [i]
end
end
rating_mapping
good[rating_mapping[2108], :]
unique(good[rating_mapping[2108], :].Rating)
using Statistics
mean(good[rating_mapping[2108], "Popularity"])
ratings = unique(good.Rating)
mean_popularities = map(ratings) do rating
indices = rating_mapping[rating]
popularities = good[indices, "Popularity"]
return mean(popularities)
end
log1pjitter(x) = log1p(x) - 0.05 + rand() / 10
scatter(ratings, mean_popularities;
xlabel="rating", ylabel="mean popularity", legend=false)
using Random
Random.seed!(1234);
scatter(log1pjitter.(agg_df.deg_ml),
log1pjitter.(agg_df.deg_web);
zcolor=agg_df.web_mean,
xlabel="degree ml", ylabel="degree web",
markersize=2, markerstrokewidth=0, markeralpha=0.8,
legend=:topleft, labels = "fraction web",
xticks=gen_ticks(maximum(classes_df.deg_ml)),
yticks=gen_ticks(maximum(classes_df.deg_web)))
import Loess
model = Loess.loess(ratings, mean_popularities);
ratings_predict = float.(sort(ratings))
popularity_predict = Loess.predict(model, ratings_predict)
# Code for fitting logistic regression model
using GLM
glm(@formula(ml_target~log1p(deg_ml)+log1p(deg_web)), classes_df, Binomial(), LogitLink())
# Code for inspecting @formula result
@formula(ml_target~log1p(deg_ml)+log1p(deg_web))
# Code for inserting columns to a data frame
df = DataFrame(x=1:3)
insertcols!(df, :y => 4:6)
insertcols!(df, :y => 4:6)
insertcols!(df, :z => 1)
insertcols!(df, 1, :a => 0)
insertcols!(df, :x, :pre_x => 2)
insertcols!(df, :x, :post_x => 3, after=true)
plot!(ratings_predict, popularity_predict; width=5, color="black")

279
ch09.jl Normal file
View File

@ -0,0 +1,279 @@
# Bogumił Kamiński, 2022
# Codes for chapter 7
# Code for section 7.1
aq = [10.0 8.04 10.0 9.14 10.0 7.46 8.0 6.58
8.0 6.95 8.0 8.14 8.0 6.77 8.0 5.76
13.0 7.58 13.0 8.74 13.0 12.74 8.0 7.71
9.0 8.81 9.0 8.77 9.0 7.11 8.0 8.84
11.0 8.33 11.0 9.26 11.0 7.81 8.0 8.47
14.0 9.96 14.0 8.1 14.0 8.84 8.0 7.04
6.0 7.24 6.0 6.13 6.0 6.08 8.0 5.25
4.0 4.26 4.0 3.1 4.0 5.39 19.0 12.50
12.0 10.84 12.0 9.13 12.0 8.15 8.0 5.56
7.0 4.82 7.0 7.26 7.0 6.42 8.0 7.91
5.0 5.68 5.0 4.74 5.0 5.73 8.0 6.89];
data = (set1=(x=aq[:, 1], y=aq[:, 2]),
set2=(x=aq[:, 3], y=aq[:, 4]),
set3=(x=aq[:, 5], y=aq[:, 6]),
set4=(x=aq[:, 7], y=aq[:, 8]));
using DataFrames
# Code for listing 7.1
aq1 = ataFrame(aq, ["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4"])
DataFrame(aq, [:x1, :y1, :x2, :y2, :x3, :y3, :x4, :y4])
# Code for creating DataFrame with automatic column names
DataFrame(aq, :auto)
# Codes for creating DataFrame from vector of vectors
aq_vec = collect(eachcol(aq))
DataFrame(aq_vec, ["x1", "y1", "x2", "y2", "x3", "y3", "x4", "y4"])
DataFrame(aq_vec, :auto)
# Codes for section 7.1.2
data.set1.x
DataFrame(x1=data.set1.x, y1=data.set1.y,
x2=data.set2.x, y2=data.set2.y,
x3=data.set3.x, y3=data.set3.y,
x4=data.set4.x, y4=data.set4.y)
DataFrame(:x1 => data.set1.x, :y1 => data.set1.y,
:x2 => data.set2.x, :y2 => data.set2.y,
:x3 => data.set3.x, :y3 => data.set3.y,
:x4 => data.set4.x, :y4 => data.set4.y)
DataFrame([:x1 => data.set1.x, :y1 => data.set1.y,
:x2 => data.set2.x, :y2 => data.set2.y,
:x3 => data.set3.x, :y3 => data.set3.y,
:x4 => data.set4.x, :y4 => data.set4.y]);
[(i, v) for i in 1:4 for v in [:x, :y]]
[string(v, i) for i in 1:4 for v in [:x, :y]]
[string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]]
DataFrame([string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]]);
data_dict = Dict([string(v, i) => getproperty(data[i], v)
for i in 1:4 for v in [:x, :y]])
collect(data_dict)
DataFrame(data_dict)
df1 = DataFrame(x1=data.set1.x)
df1.x1 === data.set1.x
df2 = DataFrame(x1=data.set1.x; copycols=false)
df2.x1 === data.set1.x
df = DataFrame(x=1:3, y=1)
df.x
DataFrame(x=[1], y=[1, 2, 3])
# Codes for section 7.1.3
data.set1
DataFrame(data.set1)
DataFrame([(a=1, b=2), (a=3, b=4), (a=5, b=6)])
data
# Code for listing 7.2
aq2 = DataFrame(data)
# Codes for listing 7.3
data_dfs = map(DataFrame, data)
# Codes for vertical concatenation examples
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4)
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4;
source="source_id")
vcat(data_dfs.set1, data_dfs.set2, data_dfs.set3, data_dfs.set4;
source="source_id"=>string.("set", 1:4))
reduce(vcat, collect(data_dfs);
source="source_id"=>string.("set", 1:4))
# Code for listing 7.4
df1 = DataFrame(a=1:3, b=11:13)
df2 = DataFrame(a=4:6, c=24:26)
vcat(df1, df2)
vcat(df1, df2; cols=:union)
# Code for listing 7.5
df_agg = DataFrame()
append!(df_agg, data_dfs.set1)
append!(df_agg, data_dfs.set2)
# Code for appending tables to a data frame
df_agg = DataFrame()
append!(df_agg, data.set1)
append!(df_agg, data.set2)
# Code for promote keyword argument
df1 = DataFrame(a=1:3, b=11:13)
df2 = DataFrame(a=4:6, b=[14, missing, 16])
append!(df1, df2)
append!(df1, df2; promote=true)
# Code for section 7.2.3
df = DataFrame()
push!(df, (a=1, b=2))
push!(df, (a=3, b=4))
df = DataFrame(a=Int[], b=Int[])
push!(df, [1, 2])
push!(df, [3, 4])
function sim_step(current)
dx, dy = rand(((1,0), (-1,0), (0,1), (0,-1)))
return (x=current.x + dx, y=current.y + dy)
end
using BenchmarkTools
@btime rand(((1,0), (-1,0), (0,1), (0,-1)));
dx, dy = (10, 20)
dx
dy
using FreqTables
using Random
Random.seed!(1234);
proptable([rand(((1,0), (-1,0), (0,1), (0,-1))) for _ in 1:10^7])
using Random
Random.seed!(6);
walk = DataFrame(x=0, y=0)
for _ in 1:10
current = walk[end, :]
push!(walk, sim_step(current))
end
walk
plot(walk.x, walk.y;
legend=false,
series_annotations=1:11,
xticks=range(extrema(walk.x)...),
yticks=range(extrema(walk.y)...))
extrema(walk.y)
range(1, 5)
(3/4)^9
# Code for listing 7.6
function walk_unique() #A
walk = DataFrame(x=0, y=0)
for _ in 1:10
current = walk[end, :]
push!(walk, sim_step(current))
end
return nrow(unique(walk)) == nrow(walk) #B
end
Random.seed!(2);
proptable([walk_unique() for _ in 1:10^5])
# Code for a note on conversion
x = [1.5]
x[1] = 1
x
# Code from section 7.3.1
Matrix(walk)
Matrix{Any}(walk)
Matrix{String}(walk)
plot(walk)
plot(Matrix(walk); labels=["x" "y"] , legend=:topleft)
# Code from section 7.3.2
Tables.columntable(walk)
using BenchmarkTools
function mysum(table)
s = 0 #A
for v in table.x #B
s += v
end
return s
end
df = DataFrame(x=1:1_000_000);
@btime mysum($df)
tab = Tables.columntable(df);
@btime mysum($tab)
@code_warntype mysum(df)
@code_warntype mysum(tab)
typeof(tab)
function barrier_mysum2(x)
s = 0
for v in x
s += v
end
return s
end
mysum2(table) = barrier_mysum2(table.x)
@btime mysum2($df)
df = DataFrame(a=[1, 1, 2], b=[1, 1, 2])
unique(df)
tab = Tables.columntable(df)
unique(tab)
# Code from section 7.3.3
Tables.rowtable(walk)
nti = Tables.namedtupleiterator(walk)
for v in nti
println(v)
end
er = eachrow(walk)
er[1]
er[end]
ec = eachcol(walk)
ec[1]
ec[end]
identity.(eachcol(walk))
df = DataFrame(x=1:2, b=["a", "b"])
identity.(eachcol(df))

284
ch10.jl Normal file
View File

@ -0,0 +1,284 @@
# Bogumił Kamiński, 2022
# Codes for chapter 8
# Codes for section 8.1
# Code for listing 8.1
import Downloads
using SHA
git_zip = "git_web_ml.zip"
if !isfile(git_zip)
Downloads.download("https://snap.stanford.edu/data/" *
"git_web_ml.zip",
git_zip)
end
isfile(git_zip)
open(sha256, git_zip) == [0x56, 0xc0, 0xc1, 0xc2,
0xc4, 0x60, 0xdc, 0x4c,
0x7b, 0xf8, 0x93, 0x57,
0xb1, 0xfe, 0xc0, 0x20,
0xf4, 0x5e, 0x2e, 0xce,
0xba, 0xb8, 0x1d, 0x13,
0x1d, 0x07, 0x3b, 0x10,
0xe2, 0x8e, 0xc0, 0x31]
# Code for opeining a zip archive
import ZipFile
git_archive = ZipFile.Reader(git_zip)
# Code for listing 8.2
function ingest_to_df(archive::ZipFile.Reader, filename::AbstractString)
idx = only(findall(x -> x.name == filename, archive.files))
return CSV.read(read(archive.files[idx]), DataFrame)
end
# Code for working with zip archive
git_archive.files
git_archive.files[2].name
findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files)
findall(x -> x.name == "", git_archive.files)
only(findall(x -> x.name == "git_web_ml/musae_git_edges.csv", git_archive.files))
only(findall(x -> x.name == "", git_archive.files))
# Code for listing 8.3
using CSV
using DataFrames
edges_df = ingest_to_df(git_archive, "git_web_ml/musae_git_edges.csv");
classes_df = ingest_to_df(git_archive, "git_web_ml/musae_git_target.csv");
close(git_archive)
summary(edges_df)
describe(edges_df, :min, :max, :mean, :nmissing, :eltype)
summary(classes_df)
describe(classes_df, :min, :max, :mean, :nmissing, :eltype)
# Code for updating data frame columns using broadcasting
edges_df .+= 1
classes_df.id .+= 1
# Code for examples of data frame broadcasting
df = DataFrame(a=1:3, b=[4, missing, 5])
df .^ 2
coalesce.(df, 0)
df .+ [10, 11, 12]
# Code for checking the order of :id column in a data frame
classes_df.id == axes(classes_df, 1)
# Code for the difference between ! and : in broadcasting assignment
df = DataFrame(a=1:3, b=1:3)
df[!, :a] .= "x"
df[:, :b] .= "x"
df
# Code for the difference between ! and : in assignment
df = DataFrame(a=1:3, b=1:3, c=1:3)
df[!, :a] = ["x", "y", "z"]
df[:, :b] = ["x", "y", "z"]
df[:, :c] = [11, 12, 13]
df
# Codes for section 8.2
# Code from listing 8.4
using Graphs
gh = SimpleGraph(nrow(classes_df))
for (from, to) in eachrow(edges_df)
add_edge!(gh, from, to)
end
gh
ne(gh)
nv(gh)
# Code for iterator destruction in iteration specification
mat = [1 2; 3 4; 5 6]
for (x1, x2) in eachrow(mat)
@show x1, x2
end
# Code for getting degrees of nodes in the graph
degree(gh)
# Code for adding a column to a data frame
classes_df.deg = degree(gh)
# Code for the difference between ! and : when adding a column
df = DataFrame()
x = [1, 2, 3]
df[!, :x1] = x
df[:, :x2] = x
df
df.x1 === x
df.x2 === x
df.x2 == x
# Code for creating a column using broadcasting
df.x3 .= 1
df
# Code for edge iterator of a graph
edges(gh)
e1 = first(edges(gh))
dump(e1)
e1.src
e1.dst
# Code for listing 8.5
function deg_class(gh, class)
deg_ml = zeros(Int, length(class))
deg_web = zeros(Int, length(class))
for edge in edges(gh)
a, b = edge.src, edge.dst
if class[b] == 1
deg_ml[a] += 1
else
deg_web[a] += 1
end
if class[a] == 1
deg_ml[b] += 1
else
deg_web[b] += 1
end
end
return (deg_ml, deg_web)
end
# Code for computing machine learning and web neighbors for gh graph
classes_df.deg_ml, classes_df.deg_web =
deg_class(gh, classes_df.ml_target)
# Code for checking type stability of deg_class function
@time deg_class(gh, classes_df.ml_target);
@code_warntype deg_class(gh, classes_df.ml_target)
# Code for checking the classes_df summary statistics
describe(classes_df, :min, :max, :mean, :std)
# Code for average degree of node in the graph
2 * ne(gh) / nv(gh)
# Code for checking correctness of computations
classes_df.deg_ml + classes_df.deg_web == classes_df.deg
# Code for showing that DataFrames.jl checks consistency of stored objects
df = DataFrame(a=1, b=11)
push!(df.a, 2)
df
# Codes for section 8.3
# Code for computing groupwise means of columns
using Statistics
for type in [0, 1], col in ["deg_ml", "deg_web"]
println((type, col, mean(classes_df[classes_df.ml_target .== type, col])))
end
gdf = groupby(classes_df, :ml_target)
combine(gdf,
:deg_ml => mean => :mean_deg_ml,
:deg_web => mean => :mean_deg_web)
using DataFramesMeta
@combine(gdf,
:mean_deg_ml = mean(:deg_ml),
:mean_deg_web = mean(:deg_web))
# Code for simple plotting of relationship between developer degree and type
using Plots
scatter(classes_df.deg_ml, classes_df.deg_web;
color=[x == 1 ? "black" : "gray" for x in classes_df.ml_target],
xlabel="degree ml", ylabel="degree web", labels=false)
# Code for aggregation of degree data
agg_df = combine(groupby(classes_df, [:deg_ml, :deg_web]),
:ml_target => (x -> 1 - mean(x)) => :web_mean)
# Code for comparison how Julia parses expressions
:ml_target => (x -> 1 - mean(x)) => :web_mean
:ml_target => x -> 1 - mean(x) => :web_mean
# Code for aggregation using DataFramesMeta.jl
@combine(groupby(classes_df, [:deg_ml, :deg_web]),
:web_mean = 1 - mean(:ml_target))
# Code for getting summary information about the aggregated data frame
describe(agg_df)
# Code for log1p function
log1p(0)
# Code for listing 8.6
function gen_ticks(maxv)
max2 = round(Int, log2(maxv))
tick = [0; 2 .^ (0:max2)]
return (log1p.(tick), tick)
end
log1pjitter(x) = log1p(x) - 0.05 + rand() / 10
using Random
Random.seed!(1234);
scatter(log1pjitter.(agg_df.deg_ml),
log1pjitter.(agg_df.deg_web);
zcolor=agg_df.web_mean,
xlabel="degree ml", ylabel="degree web",
markersize=2, markerstrokewidth=0, markeralpha=0.8,
legend=:topleft, labels = "fraction web",
xticks=gen_ticks(maximum(classes_df.deg_ml)),
yticks=gen_ticks(maximum(classes_df.deg_web)))
# Code for fitting logistic regression model
using GLM
glm(@formula(ml_target~log1p(deg_ml)+log1p(deg_web)), classes_df, Binomial(), LogitLink())
# Code for inspecting @formula result
@formula(ml_target~log1p(deg_ml)+log1p(deg_web))
# Code for inserting columns to a data frame
df = DataFrame(x=1:3)
insertcols!(df, :y => 4:6)
insertcols!(df, :y => 4:6)
insertcols!(df, :z => 1)
insertcols!(df, 1, :a => 0)
insertcols!(df, :x, :pre_x => 2)
insertcols!(df, :x, :post_x => 3, after=true)

18
chXXX_client.jl Normal file
View File

@ -0,0 +1,18 @@
using HTTP
using JSON3
using DataFrames
using Plots
df = DataFrame(K=30:2:80, max_time=0.25)
df.data = map(df.K, df.max_time) do K, max_time
@show K
@time req = HTTP.request("POST", "http://127.0.0.1:8000",
["Content-Type" => "application/json"],
JSON3.write((;K, max_time)))
return JSON3.read(req.body)
end
@assert all(==("OK"), getproperty.(df.data, :status))
df2 = select(df, :K, :data => ByRow(x -> x.value) => AsTable)
plot(plot(df2.K, df2.mv; legend=false, xlabel="K", ylabel="expected value"),
plot(df2.K, df2.zero; legend=false, xlabel="K", ylabel="probability of zero"))

45
chXXX_server.jl Normal file
View File

@ -0,0 +1,45 @@
using Genie
using Statistics
using ThreadsX
function v_asian_sample(T, X0, K, r, sd, m)::Float64
X = X0
sumX = X
d = T / m
for i in 1:m
X *= exp((r-sd^2/2)*d + sd*sqrt(d)*randn())
sumX += X
end
return exp(-r*T) * max(sumX / (m + 1) - K, 0)
end
function v_asian_value(T, X0, K, r, sd, m, max_time)
result = Float64[]
start_time = time()
while time() - start_time < max_time
append!(result, ThreadsX.map(_ -> v_asian_sample(T, X0, K, r, sd, m), 1:10_000))
end
n = length(result)
mv = mean(result)
sdv = std(result)
lo95 = mv - 1.96 * sdv / sqrt(n)
hi95 = mv + 1.96 * sdv / sqrt(n)
zero = mean(==(0), result)
return (; n, mv, sdv, lo95, hi95, zero)
end
Genie.config.run_as_server = true
Genie.Router.route("/", method=POST) do
message = Genie.Requests.jsonpayload()
return try
K = float(message["K"])
max_time = float(message["max_time"])
value = v_asian_value(1.0, 50.0, K, 0.05, 0.3, 200, max_time)
Genie.Renderer.Json.json((status="OK", value=value))
catch
Genie.Renderer.Json.json((status="ERROR", value=""))
end
end
Genie.startup()