html_md.sh

#!/bin/bash

### DOWNLOAD AND CONVERT DOCUMENTATION
# autor: kru0052
# version: 1.1
# change: converted files moved to new directory with images, deleted witch for info (-i) and deleting files (-d)
# bugs: bad formatting tables, bad links for images and other files, stayed a few html elements
###

if [ "$1" = "-t" ]; then
	# testing new function
	
	echo "Testing..."
	
fi
if [ "$1" = "-w" ]; then
	# download html pages 
	wget -X pbspro-documentation,changelog,whats-new,portal_css,portal_javascripts,++resource++jquery-ui-themes,anselm-cluster-documentation/icon.jpg -R favicon.ico,pdf.png,logo.png,background.png,application.png,search_icon.png,png.png,sh.png,touch_icon.png,anselm-cluster-documentation/icon.jpg,*js,robots.txt,*xml,RSS,download_icon.png,pdf,*zip,*rar,@@*,anselm-cluster-documentation/icon.jpg.1 --mirror --convert-links --adjust-extension --page-requisites  --no-parent https://docs.it4i.cz;
fi
if [ "$1" = "-c" ]; then
	### convert html to md
	find . -name "*.ht*" | 
	while read i; 
	do 
		# first filtering html 
		echo "$i"; 
		printf "\t\tfiltering html files...\n";
		
		HEAD=$(grep -n -m1 '<h1' "$i" |cut -f1 -d: | tr --delete '\n')
		END=$(grep -n -m1 '<!-- <div tal:content=' "$i" |cut -f1 -d: | tr --delete '\n')
		LAST=$(wc -l "$i" | cut -f1 -d' ')
		DOWN=$((LAST-END+2))

		sed '1,'"$((HEAD-1))"'d' "$i" | sed -n -e :a -e '1,'"$DOWN"'!{P;N;D;};N;ba' > "${i%.*}TMP.html"	

		# converted .html to .md
		printf "\t\t.html -> .md\n"
		pandoc -f html -t markdown+pipe_tables-grid_tables "${i%.*}TMP.html" -o "${i%.*}.md"; 
		rm "${i%.*}TMP.html";

		# create filter_auto (css, html, ...)
		grep -o -P '(?<={).*(?=})' "${i%.*}.md" | sort -u | sed '/{/d' | sed '/\$/d' >> filter_auto;
		sort -u filter_auto -o filter_auto; 

		# exceptions filter_auto 
		(while read y; 
		do 
			# search and delete according with filter_auto
			sed -e 's/'"$y"'//g' filter_auto > filter_autoTMP;
			cat filter_autoTMP > filter_auto;
		done) < ./source/exceptions_filter_auto
	
		# text filtering of html, css, ...
		printf "\t\tautomatic filter...\n"
		(while read y; 
		do 
			# search and delete according with filter_auto and erase unwanted elements
			sed -e 's/{'"$y"'}//g' "${i%.*}.md" | sed -e 's/\\//g' | sed -e 's/^: //g' | sed -e 's/<\/div>//g' | sed '/^<div/d' | sed -e 's/^Obsah//g' | sed -e 's/{#putty---before-we-start-//g' | sed -e 's/ssh-connection style="text-alignstart; "}//g'| awk -v RS='\n\n\n\n\n' 1 > "${i%.*}TMP.md";
			cat "${i%.*}TMP.md" > "${i%.*}.md";
		done) < filter_auto

		# delete temporary files
		rm "${i%.*}TMP.md";		
	done
	
	# delete empty files, duplicate files and unwanted files
	find -type f -size -10c | 
	while read i; 
	do
		rm "$i"; 	
		echo "$i deleted"; 
	done
	rm filter_autoTMP
	rm filter_auto

	(while read i; 
	do		
		if [ -f "$i" ];
		then
		   rm "$i";
		fi
		
	done) < ./source/list_rm.txt
	
	### create new folder and move converted files
	# erasing the previous transfer
	rm -rf converted;
	rm -rf info;
	
	# create folder info and view all files and folder
	mkdir info;
	find ./docs.it4i.cz -name "*.png" -type f> ./info/list_png.txt;
	find ./docs.it4i.cz -name "*.jpg" -type f> ./info/list_jpg.txt;
	find ./docs.it4i.cz -name "*.jpeg" -type f>> ./info/list_jpg.txt;
	find ./docs.it4i.cz -name "*.md" -type f> ./info/list_md.txt; 
	find ./docs.it4i.cz -type d | sort > ./info/list_folder.txt
	
	# exists file for move?	
	if [ -f ./info/list_md.txt ];
	then  	
		mkdir converted;
		(while read i; 
		do		
			mkdir "./converted/$i";
		done) < ./source/list_folder.txt
	
		# move md files to new folders
		while read a b ; do mv "$a" "./converted/$b"; done < <(paste ./info/list_md.txt ./source/list_md_mv.txt)
		# copy jpg and jpeg to new folders
		while read a b ; do cp "$a" "./converted/$b"; done < <(paste ./info/list_jpg.txt ./source/list_jpg_mv.txt)
		# copy png files to new folders
		while read a b ; do cp "$a" "./converted/$b"; done < <(paste ./info/list_png.txt ./source/list_png_mv.txt)
	else
	   echo "list_md.txt not exists!!!!!"
	fi
fi