html_md.sh

#!/bin/bash

### DOWNLOAD AND CONVERT DOCUMENTATION
# autor: kru0052
# version: 0.36
# change: repair images bugs and add new filtering html and css elements
# bugs: bad formatting tables, bad links for other files, stayed a few html elements, formatting bugs...
###

if [ "$1" = "-t" ]; then
	# testing new function
	
	echo "Testing 1..."
	
	rm -rf ./converted 
	
	# exists file for move?	
	if [ -f ./info/list_md.txt ];
	then  	
		mkdir converted;
		(while read i; 
		do		
			mkdir "./converted/$i";
		done) < ./source/list_folder.txt
	
		# move md files to new folders
		while read a b ; do 
			cp "$a" "./converted/$b"; 
			
		done < <(paste ./info/list_md.txt ./source/list_md_mv.txt)
		# copy jpg and jpeg to new folders
		
		#cat "${i%.*}TMP.md" > "${i%.*}.md";
		
		while read a b ; do cp "$a" "./converted/$b"; done < <(paste ./info/list_image.txt ./source/list_image_mv.txt)
		
		cp ./docs.it4i.cz/salomon/salomon ./converted/docs.it4i.cz/salomon/salomon
		cp ./docs.it4i.cz/salomon/salomon-2 ./converted/docs.it4i.cz/salomon/salomon-2
		cp ./converted/docs.it4i.cz/salomon/resource-allocation-and-job-execution/fairshare_formula.png ./converted/docs.it4i.cz/anselm-cluster-documentation/resource-allocation-and-job-execution/fairshare_formula.png
		cp ./converted/docs.it4i.cz/salomon/resource-allocation-and-job-execution/job_sort_formula.png ./converted/docs.it4i.cz/anselm-cluster-documentation/resource-allocation-and-job-execution/job_sort_formula.png
		cp ./converted/docs.it4i.cz/salomon/software/debuggers/vtune-amplifier.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/debuggers/vtune-amplifier.png
		cp ./converted/docs.it4i.cz/salomon/software/debuggers/Snmekobrazovky20160708v12.33.35.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/debuggers/Snmekobrazovky20160708v12.33.35.png
		
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/virtualization/virtualization-job-workflow
		mv ./virtualization-job-workflow ./converted/docs.it4i.cz/anselm-cluster-documentation/software/ 
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig6.png
		mv ./fig6.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig6.png
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig5.png
		mv ./fig5.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig5.png
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig1.png
		mv ./fig1.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig1.png
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig2.png
		mv ./fig2.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig2.png
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig4.png
		mv ./fig4.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig4.png
		wget https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig3.png
		mv ./fig3.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/fig3.png
		
		
		
	else
	   echo "list_md.txt not exists!!!!!"
	fi

	
fi
if [ "$1" = "-t1" ]; then
	# testing new function
	
	echo "Testing 1..."
	
		
		while read a ; do 
			
			echo "$a";
			 
			sed -e 's/``` /```/' "./converted/$a" | sed -e 's/  //' | sed -e 's/<span class="pln">//' | sed -e 's/<span//' | sed -e 's/class="pln">//' | sed -e 's/<\/span>//' | sed -e 's/^\*\*//' | sed -e 's/\^\[>\[1<span>\]<\/span>\]//' > "./converted/${a%.*}TMP.md";
		
			while read x ; do 
				arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
				arg2=`echo $x | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
			#echo "$arg1"; 
			#echo ">$arg2";
		
				sed -e 's/'"$arg1"'/'"$arg2"'/' "./converted/${a%.*}TMP.md" > "./converted/${a%TMP.*}.TEST.md";
				cat "./converted/${a%TMP.*}.TEST.md" > "./converted/${a%.*}TMP.md";
			done < ./source/replace.txt 
			cat "./converted/${a%.*}TMP.md" > "./converted/${a%.*}.md";
			rm "./converted/${a%.*}TMP.md";
			rm "./converted/${a%TMP.*}.TEST.md";
		done <./source/list_md_mv.txt 
		

	
fi
if [ "$1" = "-w" ]; then
	# download html pages 
	wget -X pbspro-documentation,changelog,whats-new,portal_css,portal_javascripts,++resource++jquery-ui-themes,anselm-cluster-documentation/icon.jpg -R favicon.ico,pdf.png,logo.png,background.png,application.png,search_icon.png,png.png,sh.png,touch_icon.png,anselm-cluster-documentation/icon.jpg,*js,robots.txt,*xml,RSS,download_icon.png,pdf,*zip,*rar,@@*,anselm-cluster-documentation/icon.jpg.1 --mirror --convert-links --adjust-extension --page-requisites  --no-parent https://docs.it4i.cz;
	
	wget http://verif.cs.vsb.cz/aislinn/doc/report.png
	mv report.png ./converted/salomon/software/debuggers/
	
fi
if [ "$1" = "-c" ]; then
	### convert html to md
	# erasing the previous transfer
	rm -rf converted;
	rm -rf info;
	
	find . -name "*.ht*" | 
	while read i; 
	do 
		# first filtering html 
		echo "$i"; 
		printf "\t\tfiltering html files...\n";
		
		HEAD=$(grep -n -m1 '<h1' "$i" |cut -f1 -d: | tr --delete '\n')
		END=$(grep -n -m1 '<!-- <div tal:content=' "$i" |cut -f1 -d: | tr --delete '\n')
		LAST=$(wc -l "$i" | cut -f1 -d' ')
		DOWN=$((LAST-END+2))

		sed '1,'"$((HEAD-1))"'d' "$i" | sed -n -e :a -e '1,'"$DOWN"'!{P;N;D;};N;ba' > "${i%.*}TMP.html"	

		# converted .html to .md
		printf "\t\t.html -> .md\n"
		pandoc -f html -t markdown+pipe_tables-grid_tables "${i%.*}TMP.html" -o "${i%.*}.md"; 
		rm "${i%.*}TMP.html";

		# create filter_auto (css, html, ...)
		grep -o -P '(?<={).*(?=})' "${i%.*}.md" | sort -u | sed '/{/d' | sed '/\$/d' >> filter_auto;
		sort -u filter_auto -o filter_auto; 

		# exceptions filter_auto 
		(while read y; 
		do 
			# search and delete according with filter_auto
			sed -e 's/'"$y"'//g' filter_auto > filter_autoTMP;
			cat filter_autoTMP > filter_auto;
		done) < ./source/exceptions_filter_auto
	
		# text filtering of html, css, ...
		printf "\t\tautomatic filter...\n"
		(while read y; 
		do 
			# search and delete according with filter_auto and erase unwanted elements
			sed -e 's/{'"$y"'}//g' "${i%.*}.md" | sed -e 's/\\//g' | sed -e 's/^: //g' | sed -e 's/<\/div>//g' | sed '/^<div/d' | sed -e 's/^Obsah//g' | sed -e 's/{#putty---before-we-start-//g' | sed -e 's/ssh-connection style="text-alignstart; "}//g'| awk -v RS='\n\n\n\n\n' 1 > "${i%.*}TMP.md";
			cat "${i%.*}TMP.md" > "${i%.*}.md";
		done) < filter_auto

		# delete temporary files
		rm "${i%.*}TMP.md";		
	done
	
	# delete empty files, duplicate files and unwanted files
	find -type f -size -10c | 
	while read i; 
	do
		rm "$i"; 	
		echo "$i deleted"; 
	done
	rm filter_autoTMP
	rm filter_auto

	(while read i; 
	do		
		if [ -f "$i" ];
		then
		   rm "$i";
		fi
		
	done) < ./source/list_rm.txt
	
	### create new folder and move converted files
	# create folder info and view all files and folder
	mkdir info;
	find ./docs.it4i.cz -name "*.png" -type f > ./info/list_image.txt;
	find ./docs.it4i.cz -name "*.jpg" -type f >> ./info/list_image.txt;
	find ./docs.it4i.cz -name "*.jpeg" -type f >> ./info/list_image.txt;
	find ./docs.it4i.cz -name "*.md" -type f> ./info/list_md.txt; 
	find ./docs.it4i.cz -type d | sort > ./info/list_folder.txt
	
	# exists file for move?	
	if [ -f ./info/list_md.txt ];
	then  	
		mkdir converted;
		(while read i; 
		do		
			mkdir "./converted/$i";
		done) < ./source/list_folder.txt
	
		# move md files to new folders
		while read a b ; do cp "$a" "./converted/$b"; done < <(paste ./info/list_md.txt ./source/list_md_mv.txt)
		# copy jpg and jpeg to new folders
		while read a b ; do cp "$a" "./converted/$b"; done < <(paste ./info/list_image.txt ./source/list_image_mv.txt)
	else
	   echo "list_md.txt not exists!!!!!"
	fi
fi