html_md.sh

#!/bin/bash

### DOWNLOAD AND CONVERT DOCUMENTATION
# autor: kru0052
# version: 0.51
# change: export pdf, epub
# bugs: bad internal links
###

if [ "$1" = "-e" ]; then
	# export to pdf/epub
   STARTTIME=$(date +%s)
	counter=1
	count=$(find ./converted -name "*.md" -type f | wc -l)
	if [ "$2" = "epub" ]; then
		find ./converted -name "*.md" | 
		while read i; 
		do 
			a=$(basename "$i")
			b=$(dirname "$i")
			c=$(pwd)
		
			echo "$(tput setaf 12)($counter/$count)$(tput setaf 11)$a => $(tput setaf 13)${a%.*}.epub$(tput setaf 15)"
			counter=$((counter+1))
			cd "$b"
				#if [ $a = "job-submission-and-execution.md" ]; then
					#!double subscripts problem !!!!!!
					#wkhtmltopdf job-submission-and-execution.md job-submission-and-execution.pdf
				#else
					pandoc ${a%.*}.md -o ${a%.*}.epub
				#fi
			cd "$c"
		done
	fi
	if [ "$2" = "pdf" ]; then
		find ./converted -name "*.md" | 
		while read i; 
		do 
			a=$(basename "$i")
			b=$(dirname "$i")
			c=$(pwd)
		
			echo "$(tput setaf 12)($counter/$count)$(tput setaf 11)$a => $(tput setaf 13)${a%.*}.pdf$(tput setaf 15)"
			#echo "$b"
			counter=$((counter+1))
			cd "$b"
				if [ $a = "job-submission-and-execution.md" ]; then
					#!double subscripts problem !!!!!!
					wkhtmltopdf job-submission-and-execution.md job-submission-and-execution.pdf
				else
					pandoc ${a%.*}.md  -t latex --latex-engine=xelatex --mathml -o ${a%.*}.pdf
				fi
			cd "$c"
		done
	fi
   ENDTIME=$(date +%s)
	echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete this task..."
fi
if [ "$1" = "-d" ]; then
	find . -name "*."$2 | 
	while read i; 
	do
		echo "$(tput setaf 9)$i deleted$(tput setaf 15)"
		rm "$i"
	done
fi
if [ "$1" = "-w" ]; then
	# download html pages 
   STARTTIME=$(date +%s)
	rm -rf docs.it4i.cz
	wget -X pbspro-documentation,changelog,whats-new,portal_css,portal_javascripts,++resource++jquery-ui-themes,anselm-cluster-documentation/icon.jpg -R favicon.ico,pdf.png,logo.png,background.png,application.png,search_icon.png,png.png,sh.png,touch_icon.png,anselm-cluster-documentation/icon.jpg,*js,robots.txt,*xml,RSS,download_icon.png,pdf,*zip,*rar,@@*,anselm-cluster-documentation/icon.jpg.1 --mirror --convert-links --adjust-extension --page-requisites  --no-parent https://docs.it4i.cz;
	
	# download images
	wget --directory-prefix=./docs.it4i.cz/ http://verif.cs.vsb.cz/aislinn/doc/report.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/virtualization/virtualization-job-workflow
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig1.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig2.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig3.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig4.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig5.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig6.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig7.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig7x.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig8.png
	wget --directory-prefix=./docs.it4i.cz/ https://docs.it4i.cz/anselm-cluster-documentation/software/omics-master-1/images/fig9.png
   ENDTIME=$(date +%s)
	echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete this task..."	
	
fi
if [ "$1" = "-c" ]; then
	### convert html to md
   STARTTIME=$(date +%s)
   if [ -d ./docs.it4i.cz ]; then
	# erasing the previous transfer
	
	rm -rf converted;
	rm -rf info;
	
	# erasing duplicate files and unwanted files
	(while read i; 
	do		
		if [ -f "$i" ];
		then
			echo "$(tput setaf 9)$i deleted";
		  	rm "$i";
		fi
		
	done) < ./source/list_rm
	
	counter=1
	count=$(find . -name "*.html" -type f | wc -l)
	
	find . -name "*.ht*" | 
	while read i; 
	do 
		# first filtering html 
		echo "$(tput setaf 12)($counter/$count)$(tput setaf 11)$i"; 
		counter=$((counter+1))
		printf "$(tput setaf 15)\t\tfiltering html files...\n";
		
		HEAD=$(grep -n -m1 '<h1' "$i" |cut -f1 -d: | tr --delete '\n')
		END=$(grep -n -m1 '<!-- <div tal:content=' "$i" |cut -f1 -d: | tr --delete '\n')
		LAST=$(wc -l "$i" | cut -f1 -d' ')
		DOWN=$((LAST-END+2))

		sed '1,'"$((HEAD-1))"'d' "$i" | sed -n -e :a -e '1,'"$DOWN"'!{P;N;D;};N;ba' > "${i%.*}TMP.html"	
		
		# converted .html to .md
		printf "\t\t.html -> .md\n"
		pandoc -f html -t markdown+pipe_tables-grid_tables "${i%.*}TMP.html" -o "${i%.*}.md"; 
		rm "${i%.*}TMP.html";

		# filtering html and css elements...
		printf "\t\tfiltering html and css elements...\n"
		sed -e 's/``` /```/' "${i%.*}.md"  | sed -e 's/<\/div>//g' | sed '/^<div/d' | sed -e 's/<\/span>//' | sed -e 's/^\*\*//' | sed -e 's/\\//g' | sed -e 's/^: //g' | sed -e 's/^Obsah//g' > "${i%.*}TMP.md";
		while read x ; do 
			arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
			arg2=`echo "$x" | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
	
			sed -e 's/'"$arg1"'/'"$arg2"'/' "${i%.*}TMP.md" > "${i%.*}TMP.TEST.md";
			cat -s "${i%.*}TMP.TEST.md" > "${i%.*}TMP.md";
		done < ./source/replace 
		
		# repair formatting...
		printf "\t\trepair formatting...\n"
		while read x ; do 
			arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
			arg2=`echo "$x" | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
	
			sed -e 's/'"$arg1"'/'"$arg2"'/' "${i%.*}TMP.md" > "${i%.*}TMP.TEST.md";
			cat -s "${i%.*}TMP.TEST.md" > "${i%.*}TMP.md";
		done < ./source/formatting 
		
		
		# repair image...
		printf "\t\trepair images...\n"
		while read x ; do 
			arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
			arg2=`echo "$x" | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
	
			sed -e 's/'"$arg1"'/'"$arg2"'/' "${i%.*}TMP.md" > "${i%.*}.md";
			cat -s "${i%.*}.md" > "${i%.*}TMP.md";
		done < ./source/repairIMG
		
		# repair tables
		printf "\t\trepair tables...\n"
		while read x ; do 
			arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
			arg2=`echo "$x" | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
	
			sed -e 's/'"$arg1"'/'"$arg2"'/' "${i%.*}TMP.md" > "${i%.*}.md";		
			sed -e :a -e '/\\$/N; s/\\\n//; ta' "${i%.*}.md" > "${i%.*}TMP.md";
		done < ./source/tab
		
		# replace tables
		printf "\t\treplace tables...\n"
		while read x ; do 
			arg1=`echo "$x" | cut -d"&" -f1 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;	
			arg2=`echo "$x" | cut -d"&" -f2 | sed 's:[]\[\^\$\.\*\/\"]:\\\\&:g'`;
	
			sed -e 's/'"$arg1"'/'"$arg2"'/' "${i%.*}TMP.md" > "${i%.*}.md";		
			cat -s "${i%.*}.md" > "${i%.*}TMP.md";
		done < ./source/tabREPLACE
		
		cat "${i%.*}TMP.md" > "${i%.*}.md";
		
		# delete temporary files
		rm "${i%.*}TMP.md";
		rm "${i%.*}TMP.TEST.md";		
	done
	
	# delete empty files
	find -type f -size -10c | 
	while read i; 
	do
		rm "$i"; 	
		echo "$(tput setaf 9)$i deleted"; 
	done

	
	### create new folder and move converted files
	# create folder info and list all files and folder
	mkdir info;
	echo "$(tput setaf 11)create folder info and file lists"; 
	find ./docs.it4i.cz -name "*.png" -type f > ./info/list_image;
	find ./docs.it4i.cz -name "*.jpg" -type f >> ./info/list_image;
	find ./docs.it4i.cz -name "*.jpeg" -type f >> ./info/list_image;
	find ./docs.it4i.cz -name "*.md" -type f> ./info/list_md; 
	find ./docs.it4i.cz -type d | sort > ./info/list_folder;
	
	count=$(find ./docs.it4i.cz -name "*.md" -type f | wc -l)
	if [ $count -eq 150 ]; then
		rm -rf ./converted 
	
		mkdir converted;
		(while read i; 
		do		
			mkdir "./converted/$i";
		done) < ./source/list_folder
	
		# move md files to new folders
		echo "$(tput setaf 11)moved md files"; 
		while read a b ; do 
			mv "$a" "./converted/$b";
		done < <(paste ./info/list_md ./source/list_md_mv)
		
		# copy jpg and jpeg to new folders
		echo "$(tput setaf 11)copy image files";
		while read a b ; do 
			cp "$a" "./converted/$b";
		done < <(paste ./info/list_image ./source/list_image_mv)
		cp ./docs.it4i.cz/salomon/salomon ./converted/docs.it4i.cz/salomon/salomon
		cp ./docs.it4i.cz/salomon/salomon-2 ./converted/docs.it4i.cz/salomon/salomon-2
		cp ./converted/docs.it4i.cz/salomon/resource-allocation-and-job-execution/fairshare_formula.png ./converted/docs.it4i.cz/anselm-cluster-documentation/resource-allocation-and-job-execution/fairshare_formula.png
		cp ./converted/docs.it4i.cz/salomon/resource-allocation-and-job-execution/job_sort_formula.png ./converted/docs.it4i.cz/anselm-cluster-documentation/resource-allocation-and-job-execution/job_sort_formula.png
		cp ./converted/docs.it4i.cz/salomon/software/debuggers/vtune-amplifier.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/debuggers/vtune-amplifier.png
		cp ./converted/docs.it4i.cz/salomon/software/debuggers/Snmekobrazovky20160708v12.33.35.png ./converted/docs.it4i.cz/anselm-cluster-documentation/software/debuggers/Snmekobrazovky20160708v12.33.35.png
		cp ./docs.it4i.cz/virtualization-job-workflow ./converted/docs.it4i.cz/anselm-cluster-documentation/software/ 
		cp ./docs.it4i.cz/anselm-cluster-documentation/anyconnecticon.jpg ./converted/docs.it4i.cz/salomon/accessing-the-cluster/anyconnecticon.jpg
		cp ./docs.it4i.cz/anselm-cluster-documentation/anyconnectcontextmenu.jpg ./converted/docs.it4i.cz/salomon/accessing-the-cluster/anyconnectcontextmenu.jpg
		cp ./docs.it4i.cz/get-started-with-it4innovations/accessing-the-clusters/graphical-user-interface/vnc/TightVNC_login.png ./converted/docs.it4i.cz/salomon/software/debuggers/TightVNC_login.png
	
		# listall files and folder /converted
		find ./converted -name "*.png" -type f > ./info/list_image_converted;
		find ./converted -name "*.jpg" -type f >> ./info/list_image_converted;
		find ./converted -name "*.jpeg" -type f >> ./info/list_image_converted;
		find ./converted -name "*.md" -type f> ./info/list_md_converted; 
		find ./converted -type d | sort > ./info/list_folder_converted;	
	
		echo "$(tput setaf 11)COMPLETED...$(tput setaf 15)"; 
	else
		printf "\n\n$(tput setaf 9)Can not create a folder converted, because the number of MD files disagrees. The converted files remain in the folder docs.it4i.cz !!!!...$(tput setaf 15)\n\n";
	fi
   else
   	printf "\n\n$(tput setaf 9)folder docs.it4i.cz not exists!!!!...$(tput setaf 15)\n\nRun html_md.sh -w\n\n";
   fi
   ENDTIME=$(date +%s)
   echo "It takes $(($ENDTIME - $STARTTIME)) seconds to complete this task..."
fi