249 lines
5.0 KiB
Julia
249 lines
5.0 KiB
Julia
# Bogumił Kamiński, 2022
|
|
|
|
# Codes for chapter 6
|
|
|
|
# Code for section 6.1
|
|
|
|
if isfile("puzzles.csv.bz2")
|
|
@info "file already present"
|
|
else
|
|
@info "fetching file"
|
|
download("https://database.lichess.org/" *
|
|
"lichess_db_puzzle.csv.bz2",
|
|
"puzzles.csv.bz2")
|
|
end
|
|
|
|
using CodecBzip2
|
|
compressed = read("puzzles.csv.bz2")
|
|
plain = transcode(Bzip2Decompressor, compressed)
|
|
|
|
open("puzzles.csv", "w") do io
|
|
println(io, "PuzzleId,FEN,Moves,Rating,RatingDeviation," *
|
|
"Popularity,NbPlays,Themes,GameUrl")
|
|
write(io, plain)
|
|
end
|
|
|
|
readlines("puzzles.csv")
|
|
|
|
# Code for section 6.2
|
|
|
|
using CSV
|
|
using DataFrames
|
|
puzzles = CSV.read("puzzles.csv", DataFrame);
|
|
|
|
CSV.read(plain, DataFrame);
|
|
|
|
compressed = nothing
|
|
plain = nothing
|
|
|
|
# Code for listing 6.1
|
|
|
|
puzzles
|
|
|
|
# Code for listing 6.2
|
|
|
|
describe(puzzles)
|
|
|
|
# Code for getting basic information about a data frame
|
|
|
|
ncol(puzzles)
|
|
|
|
nrow(puzzles)
|
|
|
|
names(puzzles)
|
|
|
|
# Code for section 6.3
|
|
|
|
puzzles.Rating
|
|
|
|
using BenchmarkTools
|
|
@benchmark $puzzles.Rating
|
|
|
|
puzzles.Rating == copy(puzzles.Rating)
|
|
|
|
puzzles.Rating === copy(puzzles.Rating)
|
|
|
|
puzzles.Rating === puzzles.Rating
|
|
|
|
copy(puzzles.Rating) === copy(puzzles.Rating)
|
|
|
|
puzzles."Rating"
|
|
|
|
col = "Rating"
|
|
|
|
data_frame_name[selected_rows, selected_columns]
|
|
|
|
puzzles[:, "Rating"]
|
|
puzzles[:, :Rating]
|
|
puzzles[:, 4]
|
|
puzzles[:, col]
|
|
|
|
columnindex(puzzles, "Rating")
|
|
|
|
columnindex(puzzles, "Some fancy column name")
|
|
|
|
hasproperty(puzzles, "Rating")
|
|
hasproperty(puzzles, "Some fancy column name")
|
|
|
|
@benchmark $puzzles[:, :Rating]
|
|
|
|
puzzles[!, "Rating"]
|
|
puzzles[!, :Rating]
|
|
puzzles[!, 4]
|
|
puzzles[!, col]
|
|
|
|
using Plots
|
|
plot(histogram(puzzles.Rating, label="Rating"),
|
|
histogram(puzzles.RatingDeviation, label="RatingDeviation"),
|
|
histogram(puzzles.Popularity, label="Popularity"),
|
|
histogram(puzzles.NbPlays, label="NbPlays"))
|
|
|
|
plot([histogram(puzzles[!, col]; label=col) for
|
|
col in ["Rating", "RatingDeviation",
|
|
"Popularity", "NbPlays"]]...)
|
|
|
|
# Code for section 6.4
|
|
|
|
using Statistics
|
|
plays_lo = median(puzzles.NbPlays)
|
|
puzzles.NbPlays .> plays_lo
|
|
|
|
puzzles.NbPlays > plays_lo
|
|
|
|
rating_lo = 1500
|
|
rating_hi = quantile(puzzles.Rating, 0.99)
|
|
rating_lo .< puzzles.Rating .< rating_hi
|
|
|
|
row_selector = (puzzles.NbPlays .> plays_lo) .&&
|
|
(rating_lo .< puzzles.Rating .< rating_hi)
|
|
|
|
sum(row_selector)
|
|
count(row_selector)
|
|
|
|
# Code for listing 6.3
|
|
|
|
good = puzzles[row_selector, ["Rating", "Popularity"]]
|
|
|
|
# Code for plotting histograms
|
|
|
|
plot(histogram(good.Rating; label="Rating"),
|
|
histogram(good.Popularity; label="Popularity"))
|
|
|
|
# Code for column selectors
|
|
|
|
puzzles[1, "Rating"]
|
|
|
|
puzzles[:, "Rating"]
|
|
|
|
row1 = puzzles[1, ["Rating", "Popularity"]]
|
|
|
|
row1["Rating"]
|
|
row1[:Rating]
|
|
row1[1]
|
|
row1.Rating
|
|
row1."Rating"
|
|
|
|
good = puzzles[row_selector, ["Rating", "Popularity"]]
|
|
|
|
good[1, "Rating"]
|
|
good[1, :]
|
|
good[:, "Rating"]
|
|
good[:, :]
|
|
|
|
names(puzzles, ["Rating", "Popularity"])
|
|
names(puzzles, [:Rating, :Popularity])
|
|
names(puzzles, [4, 6])
|
|
names(puzzles, [false, false, false, true, false, true, false, false, false])
|
|
names(puzzles, r"Rating")
|
|
names(puzzles, Not([4, 6]))
|
|
names(puzzles, Not(r"Rating"))
|
|
names(puzzles, Between("Rating", "Popularity"))
|
|
names(puzzles, :)
|
|
names(puzzles, All())
|
|
names(puzzles, Cols(r"Rating", "NbPlays"))
|
|
names(puzzles, Cols(startswith("P")))
|
|
|
|
names(puzzles, startswith("P"))
|
|
|
|
names(puzzles, Real)
|
|
|
|
names(puzzles, AbstractString)
|
|
|
|
puzzles[:, names(puzzles, Real)]
|
|
|
|
# Code for row subsetting
|
|
|
|
df1 = puzzles[:, ["Rating", "Popularity"]];
|
|
df2 = puzzles[!, ["Rating", "Popularity"]];
|
|
|
|
df1 == df2
|
|
df1 == puzzles
|
|
df2 == puzzles
|
|
|
|
df1.Rating === puzzles.Rating
|
|
df1.Popularity === puzzles.Popularity
|
|
df2.Rating === puzzles.Rating
|
|
df2.Popularity === puzzles.Popularity
|
|
|
|
@benchmark $puzzles[:, ["Rating", "Popularity"]]
|
|
@benchmark $puzzles[!, ["Rating", "Popularity"]]
|
|
|
|
puzzles[1, 1]
|
|
puzzles[[1], 1]
|
|
puzzles[1, [1]]
|
|
puzzles[[1], [1]]
|
|
|
|
# Code for making views
|
|
|
|
@view puzzles[1, 1]
|
|
|
|
@view puzzles[[1], 1]
|
|
|
|
@view puzzles[1, [1]]
|
|
|
|
@view puzzles[[1], [1]]
|
|
|
|
@btime $puzzles[$row_selector, ["Rating", "Popularity"]];
|
|
@btime @view $puzzles[$row_selector, ["Rating", "Popularity"]];
|
|
|
|
parentindices(@view puzzles[row_selector, ["Rating", "Popularity"]])
|
|
|
|
# Code for section 6.5
|
|
|
|
describe(good)
|
|
|
|
rating_mapping = Dict{Int, Vector{Int}}()
|
|
for (i, rating) in enumerate(good.Rating)
|
|
if haskey(rating_mapping, rating)
|
|
push!(rating_mapping[rating], i)
|
|
else
|
|
rating_mapping[rating] = [i]
|
|
end
|
|
end
|
|
rating_mapping
|
|
|
|
good[rating_mapping[2108], :]
|
|
|
|
unique(good[rating_mapping[2108], :].Rating)
|
|
|
|
using Statistics
|
|
mean(good[rating_mapping[2108], "Popularity"])
|
|
|
|
ratings = unique(good.Rating)
|
|
|
|
mean_popularities = map(ratings) do rating
|
|
indices = rating_mapping[rating]
|
|
popularities = good[indices, "Popularity"]
|
|
return mean(popularities)
|
|
end
|
|
|
|
scatter(ratings, mean_popularities;
|
|
xlabel="rating", ylabel="mean popularity", legend=false)
|
|
|
|
import Loess
|
|
model = Loess.loess(ratings, mean_popularities);
|
|
ratings_predict = float.(sort(ratings))
|
|
popularity_predict = Loess.predict(model, ratings_predict)
|
|
|
|
plot!(ratings_predict, popularity_predict; width=5, color="black")
|