fixed cleanup script
This commit is contained in:
N_T
parent
2db1f59d24
commit
1548db1731
File diff suppressed because one or more lines are too long
@ -8,152 +8,154 @@ pdfMode = True
|
||||
|
||||
print(format(sys.argv))
|
||||
if len(sys.argv)>1:
|
||||
if int(sys.argv[1])>0:
|
||||
print("WWW mode on")
|
||||
pdfMode = False
|
||||
if int(sys.argv[1])>0:
|
||||
print("WWW mode on")
|
||||
pdfMode = False
|
||||
|
||||
fileList = [
|
||||
"diffphys-code-burgers.ipynb", "diffphys-code-ns.ipynb", "diffphys-code-sol.ipynb", "physicalloss-code.ipynb", # TF
|
||||
"bayesian-code.ipynb", "supervised-airfoils.ipynb", # pytorch
|
||||
"reinflearn-code.ipynb", # phiflow
|
||||
"physgrad-comparison.ipynb", # jax
|
||||
"physgrad-code.ipynb", # pip
|
||||
]
|
||||
"diffphys-code-burgers.ipynb", "diffphys-code-ns.ipynb", "diffphys-code-sol.ipynb", "physicalloss-code.ipynb", # TF
|
||||
"bayesian-code.ipynb", "supervised-airfoils.ipynb", # pytorch
|
||||
"reinflearn-code.ipynb", # phiflow
|
||||
"physgrad-comparison.ipynb", # jax
|
||||
"physgrad-code.ipynb", # pip
|
||||
]
|
||||
|
||||
#fileList = [ "physgrad-code.ipynb"] # debug, only 1 file
|
||||
#fileList = [ "t1.ipynb" ] # debug
|
||||
#fileList = [ "diffphys-code-sol.ipynb" ] # debug
|
||||
|
||||
|
||||
# main
|
||||
|
||||
for fnOut in fileList:
|
||||
if not os.path.isfile(fnOut):
|
||||
print("Error: "+fnOut+" not found!"); exit(1)
|
||||
if not os.path.isfile(fnOut):
|
||||
print("Error: "+fnOut+" not found!"); exit(1)
|
||||
|
||||
# create backups
|
||||
fn0 = fnOut[:-5] + "bak"
|
||||
fn = fn0 + "0"; cnt = 0
|
||||
while os.path.isfile(fn):
|
||||
#print("Error: "+fn+" already exists!"); exit(1)
|
||||
print("Warning: "+fn+" already exists!")
|
||||
fn = fn0 + format(cnt); cnt=cnt+1
|
||||
# create backups
|
||||
fn0 = fnOut[:-5] + "bak"
|
||||
fn = fn0 + "0"; cnt = 0
|
||||
while os.path.isfile(fn):
|
||||
#print("Error: "+fn+" already exists!"); exit(1)
|
||||
print("Warning: "+fn+" already exists!")
|
||||
fn = fn0 + format(cnt); cnt=cnt+1
|
||||
|
||||
print("renaming "+fnOut+ " to "+fn )
|
||||
if os.path.isfile(fnOut):
|
||||
os.rename(fnOut, fn)
|
||||
if not os.path.isfile(fn):
|
||||
print("Error: "+fn+" missing!")
|
||||
exit(1)
|
||||
print("renaming "+fnOut+ " to "+fn )
|
||||
if os.path.isfile(fnOut):
|
||||
os.rename(fnOut, fn)
|
||||
if not os.path.isfile(fn):
|
||||
print("Error: "+fn+" missing!")
|
||||
exit(1)
|
||||
|
||||
with open(fn) as file:
|
||||
d = json.load(file)
|
||||
with open(fn) as file:
|
||||
d = json.load(file)
|
||||
|
||||
#print(d.keys()) #print(d["cells"][0].keys())
|
||||
#print(d.keys()) #print(d["cells"][0].keys())
|
||||
|
||||
# remove TF / pytorch warnings, build list of regular expressions to search for
|
||||
# double check, redundant with removing stderr cells (cf delE)
|
||||
res = []
|
||||
res.append( re.compile(r"WARNING:tensorflow:") )
|
||||
res.append( re.compile(r"UserWarning:") )
|
||||
res.append( re.compile(r"DeprecationWarning:") )
|
||||
res.append( re.compile(r"InsecureRequestWarning") ) # for https download
|
||||
res.append( re.compile(r"Building wheel") ) # phiflow install, also gives weird unicode characters
|
||||
res.append( re.compile(r"warnings.warn") ) # phiflow warnings
|
||||
res.append( re.compile(r"WARNING:absl") ) # jax warnings
|
||||
# remove TF / pytorch warnings, build list of regular expressions to search for
|
||||
# double check, redundant with removing stderr cells (cf delE)
|
||||
res = []
|
||||
res.append( re.compile(r"WARNING:tensorflow:") )
|
||||
res.append( re.compile(r"UserWarning:") )
|
||||
res.append( re.compile(r"DeprecationWarning:") )
|
||||
res.append( re.compile(r"InsecureRequestWarning") ) # for https download
|
||||
res.append( re.compile(r"Building wheel") ) # phiflow install, also gives weird unicode characters
|
||||
res.append( re.compile(r"warnings.warn") ) # phiflow warnings
|
||||
res.append( re.compile(r"WARNING:absl") ) # jax warnings
|
||||
|
||||
res.append( re.compile(r"ERROR: pip") ) # pip dependencies
|
||||
res.append( re.compile(r"requires imgaug") ) # pip dependencies
|
||||
res.append( re.compile(r"See the documentation of nn.Upsample") ) # pip dependencies
|
||||
res.append( re.compile(r"ERROR: pip") ) # pip dependencies
|
||||
res.append( re.compile(r"requires imgaug") ) # pip dependencies
|
||||
res.append( re.compile(r"See the documentation of nn.Upsample") ) # pip dependencies
|
||||
|
||||
# remove all "warnings.warn" from phiflow?
|
||||
# remove all "warnings.warn" from phiflow?
|
||||
|
||||
# shorten data line: "0.008612174447657694, 0.02584669669548606, 0.043136357266407785"
|
||||
reD = re.compile(r"\[0.008612174447657694, 0.02584669669548606, 0.043136357266407785.+\]" )
|
||||
reDt = "[0.008612174447657694, 0.02584669669548606, 0.043136357266407785 ... ]"
|
||||
# shorten data line: "0.008612174447657694, 0.02584669669548606, 0.043136357266407785"
|
||||
reD = re.compile(r"\[0.008612174447657694, 0.02584669669548606, 0.043136357266407785.+\]" )
|
||||
reDt = "[0.008612174447657694, 0.02584669669548606, 0.043136357266407785 ... ]"
|
||||
|
||||
t="cells"
|
||||
okay = 0
|
||||
deletes = 0
|
||||
for i in range(len(d[t])):
|
||||
#for i in range(len(d[t])):
|
||||
#print(d[t][0]["cell_type"])
|
||||
#print(d[t][i]["cell_type"])
|
||||
t="cells"
|
||||
okay = 0
|
||||
deletes = 0
|
||||
for i in range(len(d[t])):
|
||||
#for i in range(len(d[t])):
|
||||
#print(d[t][0]["cell_type"])
|
||||
#print(d[t][i]["cell_type"])
|
||||
|
||||
# remove images after code
|
||||
# remove images after code
|
||||
|
||||
if d[t][i]["cell_type"]=="code":
|
||||
#print(d[t][i].keys())
|
||||
#d[t][i]["outputs"] = ""
|
||||
#print(d[t][i]["outputs"])
|
||||
if d[t][i]["cell_type"]=="code":
|
||||
#print(d[t][i].keys())
|
||||
#d[t][i]["outputs"] = ""
|
||||
#print(d[t][i]["outputs"])
|
||||
|
||||
if pdfMode:
|
||||
for j in range(len( d[t][i]["source"] )):
|
||||
#print( d[t][i]["source"][j] )
|
||||
#print( type(d[t][i]["source"][j] ))
|
||||
dsOut = reD.sub( reDt, d[t][i]["source"][j] ) # replace long number string (only for burgers)
|
||||
d[t][i]["source"][j] = dsOut
|
||||
deletes = deletes+1
|
||||
#print( d[t][i]["source"][j] +"\n >>> \n" +d2 )
|
||||
if pdfMode:
|
||||
for j in range(len( d[t][i]["source"] )):
|
||||
#print( d[t][i]["source"][j] )
|
||||
#print( type(d[t][i]["source"][j] ))
|
||||
dsOut = reD.sub( reDt, d[t][i]["source"][j] ) # replace long number string (only for burgers)
|
||||
d[t][i]["source"][j] = dsOut
|
||||
deletes = deletes+1
|
||||
#print( d[t][i]["source"][j] +"\n >>> \n" +d2 )
|
||||
|
||||
delE = [] # collect whole entries (sections) to delete
|
||||
delE = [] # collect whole entries (sections) to delete
|
||||
|
||||
#print(len( d[t][i]["outputs"] ))
|
||||
for j in range(len( d[t][i]["outputs"] )):
|
||||
#print(type( d[t][i]["outputs"][j] ))
|
||||
#print( d[t][i]["outputs"][j].keys() )
|
||||
#print(len( d[t][i]["outputs"] ))
|
||||
for j in range(len( d[t][i]["outputs"] )):
|
||||
#print(type( d[t][i]["outputs"][j] ))
|
||||
#print( d[t][i]["outputs"][j].keys() )
|
||||
|
||||
# search for error stderr cells
|
||||
if d[t][i]["outputs"][j]["output_type"]=="stream":
|
||||
#print("output j name: "+ format( d[t][i]["outputs"][j]["name"] ) )
|
||||
#print("output j: "+ format( d[t][i]["outputs"][j] ) )
|
||||
if d[t][i]["outputs"][j]["name"]=="stderr":
|
||||
print("stderr found! len text "+ format(len( d[t][i]["outputs"][j]["text"]) ) +", removing entry "+format(j) )
|
||||
delE.append(j) # remove the whole stderr entry
|
||||
# search for error stderr cells
|
||||
if d[t][i]["outputs"][j]["output_type"]=="stream":
|
||||
#print("output j name: "+ format( d[t][i]["outputs"][j]["name"] ) )
|
||||
#print("output j: "+ format( d[t][i]["outputs"][j] ) )
|
||||
if d[t][i]["outputs"][j]["name"]=="stderr":
|
||||
print("stderr found! len text "+ format(len( d[t][i]["outputs"][j]["text"]) ) +", removing entry "+format(j) )
|
||||
delE.append(j) # remove the whole stderr entry
|
||||
|
||||
# images
|
||||
if d[t][i]["outputs"][j]["output_type"]=="stream":
|
||||
#print("len "+ format(len( d[t][i]["outputs"][j]["text"] )) )
|
||||
# images
|
||||
if d[t][i]["outputs"][j]["output_type"]=="stream":
|
||||
#print("len "+ format(len( d[t][i]["outputs"][j]["text"] )) )
|
||||
|
||||
dell = [] # collect lines to delete
|
||||
for k in range( len( d[t][i]["outputs"][j]["text"] ) ):
|
||||
#print(" tout "+ d[t][i]["outputs"][j]["text"][k] ) # debug , print all lines
|
||||
nums = []; all_good = True
|
||||
for rr in range(len(res)):
|
||||
nums.append( res[rr].search( d[t][i]["outputs"][j]["text"][k] ) )
|
||||
if nums[-1] is not None:
|
||||
all_good = False # skip!
|
||||
dell = [] # collect lines to delete
|
||||
for k in range( len( d[t][i]["outputs"][j]["text"] ) ):
|
||||
#print(" tout "+ d[t][i]["outputs"][j]["text"][k] ) # debug , print all lines - ACTIVATE to LOCATE errors
|
||||
nums = []; all_good = True
|
||||
for rr in range(len(res)):
|
||||
nums.append( res[rr].search( d[t][i]["outputs"][j]["text"][k] ) )
|
||||
if nums[-1] is not None:
|
||||
all_good = False # skip!
|
||||
|
||||
if all_good:
|
||||
okay = okay+1
|
||||
else: # delete line "dell"
|
||||
deletes = deletes+1
|
||||
dell.append(d[t][i]["outputs"][j]["text"][k])
|
||||
#print( format(nums) +" " + d[t][i]["outputs"][j]["text"][k] ) # len( d[t][i]["outputs"][j]["text"][k] ) )
|
||||
if all_good:
|
||||
okay = okay+1
|
||||
else: # delete line "dell"
|
||||
deletes = deletes+1
|
||||
dell.append(d[t][i]["outputs"][j]["text"][k])
|
||||
#print( format(nums) +" " + d[t][i]["outputs"][j]["text"][k] ) # len( d[t][i]["outputs"][j]["text"][k] ) )
|
||||
|
||||
for dl in dell:
|
||||
d[t][i]["outputs"][j]["text"].remove(dl)
|
||||
#print("len after "+format( len( d[t][i]["outputs"][j]["text"] )) + " A") # debug
|
||||
for dl in dell:
|
||||
d[t][i]["outputs"][j]["text"].remove(dl)
|
||||
#print("len after "+format( len( d[t][i]["outputs"][j]["text"] )) + " A") # debug
|
||||
|
||||
# afterwards (potentially remove whole entries)
|
||||
# if len(delE)>0:
|
||||
# print("len bef "+format( len( d[t][i]["outputs"] )) + " A " + format(delE)) # debug
|
||||
for de in delE:
|
||||
#print(type(d[t][i]["outputs"])); print(de)
|
||||
d[t][i]["outputs"].pop(de) # remove array element
|
||||
deletes+=1
|
||||
# if len(delE)>0:
|
||||
# print("len after "+format( len( d[t][i]["outputs"] )) + " A") # debug
|
||||
# afterwards (potentially remove whole entries)
|
||||
if len(delE)>0:
|
||||
delE.sort(reverse=True)
|
||||
#print("len bef "+format( len( d[t][i]["outputs"] )) + " A " + format(delE)) # debug
|
||||
|
||||
for de in delE:
|
||||
#print(type(d[t][i]["outputs"])); print(de)
|
||||
d[t][i]["outputs"].pop(de) # remove array element
|
||||
deletes+=1
|
||||
# if len(delE)>0:
|
||||
# print("len after "+format( len( d[t][i]["outputs"] )) + " A") # debug
|
||||
|
||||
if deletes==0:
|
||||
print("Warning: Nothing found in "+fn+"!")
|
||||
if not os.path.isfile(fnOut):
|
||||
os.rename(fn, fnOut)
|
||||
else:
|
||||
print("Error, both files exist!?")
|
||||
exit(1)
|
||||
if deletes==0:
|
||||
print("Warning: Nothing found in "+fn+"!")
|
||||
if not os.path.isfile(fnOut):
|
||||
os.rename(fn, fnOut)
|
||||
else:
|
||||
print("Error, both files exist!?")
|
||||
exit(1)
|
||||
|
||||
else:
|
||||
print(" ... writing "+fnOut )
|
||||
with open(fnOut,'w') as fileOut:
|
||||
json.dump(d,fileOut, indent=1, sort_keys=True)
|
||||
else:
|
||||
print(" ... writing "+fnOut )
|
||||
with open(fnOut,'w') as fileOut:
|
||||
json.dump(d,fileOut, indent=1, sort_keys=True)
|
||||
|
||||
|
||||
17
make-pdf.sh
17
make-pdf.sh
@ -10,7 +10,6 @@ echo
|
||||
echo WARNING - still requires one manual quit of first pdf/latex pass, use shift-x to quit
|
||||
echo
|
||||
|
||||
PYT=python3.7
|
||||
PYT=python3
|
||||
|
||||
# warning - modifies notebooks!
|
||||
@ -18,9 +17,21 @@ ${PYT} json-cleanup-for-pdf.py
|
||||
|
||||
# clean / remove _build dir ?
|
||||
|
||||
# GEN!
|
||||
/Users/thuerey/Library/Python/3.9/bin/jupyter-book build .
|
||||
xelatex book
|
||||
|
||||
exit # sufficient for newer jupyter book versions
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# old "pre" GEN
|
||||
#/Users/thuerey/Library/Python/3.7/bin/jupyter-book build . --builder pdflatex
|
||||
/Users/thuerey/Library/Python/3.9/bin/jupyter-book build . --builder pdflatex
|
||||
#/Users/thuerey/Library/Python/3.9/bin/jupyter-book build . --builder pdflatex
|
||||
|
||||
# old cleanup
|
||||
|
||||
cd _build/latex
|
||||
#mv book.pdf book-xetex.pdf # not necessary, failed anyway
|
||||
|
||||
Loading…
Reference in New Issue
Block a user