Compare commits

..

No commits in common. "master" and "v0.1" have entirely different histories.
master ... v0.1

22 changed files with 462 additions and 283 deletions

3
.gitignore vendored
View File

@ -1,5 +1,2 @@
CONFIG
*.pyc
build
dist
*.egg-info

37
bbbib.sh Executable file
View File

@ -0,0 +1,37 @@
#!/bin/bash
WHERE=$( cd $(dirname $0) ; pwd -P )
# default configs
if [[ -e "$WHERE/CONFIG" ]]; then
source $WHERE/CONFIG
fi
OPTIND=1 #reset getopts
while getopts :b: opts; do
case $opts in
b)
BIB=$OPTARG
;;
?)
echo "invalid option -$OPTARG";
exit;
;;
:)
echo "option -$OPTARG requires an argument";
exit;
;;
esac
done
shift $((OPTIND-1))
# bibliography exists?
if [[ -z "$BIB" ]]; then
echo "No bibliography"
exit;
fi
./parse/refactorbib.py -i $BIB
#node $BIBLIOGRAPHE_PATH/refactorbib.js --data $BIB --index --print

View File

@ -1,27 +0,0 @@
#!/usr/bin/env python
import os, sys, glob
from nnnotes import parse
if __name__ == '__main__':
indexfile = '.indx'
notes = glob.glob('*.mmd')
if len(notes) > 1:
sys.exit('More or less *.mmd files than expected.')
elif len(notes) < 1:
sys.exit('No *.mmd in current directory.')
try:
note = open(notes[0], 'r+')
except:
sys.exit('Cannot open ' + notes[0])
with open(indexfile, 'w') as indx:
parse.run(note, indx)

View File

@ -1,40 +0,0 @@
#!/usr/bin/env python
import os, sys, glob, json
from nnnotes import compare, inject
if __name__ == '__main__':
indexfile = '.indx'
pdfs = glob.glob('*.pdf')
if len(pdfs) > 1:
sys.exit('More than one pdf in current directory. No obvious choice. Aborting.')
if len(pdfs) == 1:
pdf = pdfs[0]
if os.path.isfile(indexfile):
tmp = '.tmp'
os.system('plfr -json ' + pdf + ' > ' + tmp) ### relying on plfr
diff = compare.run(indexfile, tmp) ### new highlights in pdf?
with open(tmp, 'w') as fptmp:
json.dump(diff, fptmp)
indexfile = tmp
else:
os.system('plfr -json ' + pdf + ' > ' + indexfile)
notes = glob.glob('*.mmd')
if len(notes) > 1:
sys.exit('More or less *.mmd files than expected.')
elif len(notes) < 1:
sys.exit('No *.mmd in current directory.')
note = notes[0]
inject.run(note, indexfile)
if os.path.isfile('.tmp'):
os.remove('.tmp')

View File

@ -1,83 +0,0 @@
#!/usr/bin/env python
import os, sys, argparse, shutil, subprocess
import getpass, time
from nnnotes import TEMPLATE_PATH
def yes_no(question):
sys.stdout.write(question)
yes = set(['yes','y', 'ye', ''])
no = set(['no','n'])
choice = raw_input().lower()
if choice in yes:
return True
elif choice in no:
return False
else:
sys.stdout.write("Please respond with 'yes' or 'no'")
def check_note_exists(title):
if os.path.isdir(title):
q = yes_no('The note already exists in the current directory.\nDo you want to proceed and erase the current note? [y/n] ')
if not q:
sys.exit('Notes already exists. Aborting.')
return True
return False
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('-t', '--title', action="store", help="title of the note", required=True)
p.add_argument('-f', '--file', action="store", help="(pdf) file to extract notes from")
p.add_argument('-b', '--bibtex', action="store", help="bibtex file containing bibliographical information")
p.add_argument('-i', '--bibtexitem', action="store", help="name of the item in the bibtex file")
p.add_argument('-s', '--style', action="store", help="bibliographic style")
args = p.parse_args()
if not check_note_exists(args.title):
# make directory for the note
os.makedirs(args.title)
# copy template files -- this needs to change....
#home = os.path.dirname(os.path.realpath(__file__))
os.system('cp -aR ' + TEMPLATE_PATH + '/* ' + args.title)
os.chdir(args.title)
# copy note file if it exists
if args.file is not None and os.path.isfile(args.file):
shutil.copy2(args.file, os.path.basename(args.file))
# style
style_arg = ''
if args.style is not None:
os.system('sed -i.bak "s#CSL\ :=#CSL\ := ' + args.style + '#g" Makefile')
style_arg = '-s ' + args.style
# bibtex item
bibtexitem_arg = ''
if args.bibtexitem is not None:
bibtexitem_arg = '-i ' + args.bibtexitem
# generate bibliographic element
if args.bibtex is not None and os.path.isfile(args.bibtex):
os.system('sed -i.bak "s#BIB\ :=#BIB\ := ' + args.bibtex + '#g" Makefile')
ref = '> ' + subprocess.check_output('csl_unsorted ' + args.bibtex + ' ' + bibtexitem_arg + ' ' + style_arg + ' -f md', shell=True).rstrip()
os.system('sed -i.bak "s~*MACHINE-REF*~' + ref + '~g ; s~%\ title~%\ ' + ref + '~g" notes.mmd')
author = getpass.getuser()
date = time.strftime("%d/%m/%Y")
os.system('sed -i.bak "s~%\ author~%\ ' + author + '~g ; s~%\ date~%\ ' + date + '~g" notes.mmd')
# cleanup
os.system('rm *.bak')
os.chdir('..')

110
nnnew.sh Executable file
View File

@ -0,0 +1,110 @@
#!/bin/bash
#Black 0;30 Dark Gray 1;30
#Blue 0;34 Light Blue 1;34
#Green 0;32 Light Green 1;32
#Cyan 0;36 Light Cyan 1;36
#Red 0;31 Light Red 1;31
#Purple 0;35 Light Purple 1;35
#Brown/Orange 0;33 Yellow 1;33
#Light Gray 0;37 White 1;37
BLUE='\033[0;34m'
NC='\033[0m'
WHERE=$( cd $(dirname $0) ; pwd -P )
# default configs
if [[ -e "$WHERE/CONFIG" ]]; then
source $WHERE/CONFIG
fi
print_usage() {
printf "usage: ${BLUE}nnnew.sh -t [title] -f [pdf file] -b [json biblio] -s [style] -r [reference]${NC}"
}
OPTIND=1 #reset getopts
while getopts t:f:b:s:r: opts; do
case $opts in
t)
TITLE=$OPTARG
;;
f)
FILE=$OPTARG
;;
b)
BIB=$OPTARG
;;
s)
STYLE=$OPTARG
;;
r)
REF=$OPTARG
;;
?)
#echo "invalid option -$OPTARG";
print_usage
exit;
;;
:)
echo "option -$OPTARG requires an argument";
print_usage
exit;
;;
esac
done
shift $((OPTIND-1))
if [[ -z "$TITLE" ]]; then
echo "No title provided. Aborting."
print_usage
exit;
fi
if [[ -d "$TITLE" ]]; then
read -p "Specified note already exists... Do you wish to continue? " yn
case $yn in
[Yy]* ) rm -r $TITLE; break;;
[Nn]* ) exit;;
* ) echo "Please answer yes or no.";;
esac
fi
printf "\n-----\ntitle: $TITLE\nfile: $FILE\nbib: $BIB\ncsl: $STYLE\nref: $REF\nbibliographe: $BIBLIOGRAPHE_PATH\n-----\n"
# output
mkdir -p $TITLE
# copy templates
cp -aR $WHERE/template/* $TITLE
cd $TITLE
touch notes.mmd
# file exists? copy to output
[ -z "$FILE" ] && echo "No file to copy. Make sure to reference in text in the note!" || curl -O $FILE
# bibliography exists? fix makefile accordingly
[ -z "$BIB" ] && echo "No bibliography" || sed -i.bak "s#BIB\ :=#BIB\ := $BIB#g" Makefile
# csl style exists? fix makefile accordingly
[ -z "$STYLE" ] && echo "No csl style" || sed -i.bak "s#CSL\ :=#CSL\ := $STYLE#g" Makefile
if [ -z "$BIBLIOGRAPHE_PATH" ]; then
echo "No bibliographe"
else
./parse/refactorbib.py $BIB > tmpbib.json
REFERENCE=$(node $BIBLIOGRAPHE_PATH/generatebib.js --data tmpbib.json --items [\"$REF\"] --output md)
AUTHOR=$(whoami)
DATE=$(date +%Y\-%m\-%d)
echo $REFERENCE
rm tmpbib.json
sed -i.bak "s~*MACHINE-REF*~$REFERENCE~g ; s~%\ title~%\ $REFERENCE~g ; s~%\ author~%\ $AUTHOR~g ; s~%\ date~%\ $DATE~g" notes.mmd
fi
# cleanup
rm *.bak
cd ..

View File

@ -1,3 +0,0 @@
import os
TEMPLATE_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'template')

View File

@ -1,8 +1,4 @@
#!/usr/bin/env python
# compares two json formatted indexes
# and return the difference
import sys, os, string, json
from operator import attrgetter
@ -12,8 +8,8 @@ fp2 = ''
def difference(data1, data2):
output = {'QUOTES' : [], 'NOTES' : []}
d1 = {i['quote'].rstrip(): i for i in data1['QUOTES']}
d2 = {i['quote'].rstrip(): i for i in data2['QUOTES']}
d1 = {i['quote']: i for i in data1['QUOTES']}
d2 = {i['quote']: i for i in data2['QUOTES']}
#create sets
s1 = set(d1.keys())
@ -51,32 +47,28 @@ def open_file(p):
sys.exit('File %s does not exists... Aborting.' % p)
return open(p, 'rb')
def run(filename1, filename2):
#open files
if __name__ == '__main__':
if len(sys.argv) < 2:
sys.exit('No input file... Aborting.')
try:
fp1 = open_file(filename1)
fp1 = open_file(sys.argv[1])
except:
sys.exit("Can't open file " + filename1 + ". Aborting.")
sys.exit("Can't open file " + sys.argv[1] + ". Aborting.")
if len(sys.argv) < 3:
fp2 = sys.stdin
else:
try:
fp2 = open_file(filename2)
fp2 = open_file(sys.argv[2])
except:
# it may be std.in
try:
fp2 = filename2
fp2.tell()
except:
sys.exit("Can't open file " + filename2 + ". Aborting.")
sys.exit("Can't open file " + sys.argv[2] + ". Aborting.")
#read data
try:
sdata = fp1.read()
data1 = json.loads(sdata)
except:
e = "<compare> Error loading data from" + filename1 + ". Aborting.\n"
e = "<compare> Error loading data from" + sys.argv[1] + ". Aborting.\n"
if sdata:
e += "Traceback: " + sdata
e += "Traceback: " + sdata1
fp2.close()
sys.exit(e)
finally:
@ -94,21 +86,11 @@ def run(filename1, filename2):
finally:
fp2.close()
#process
data = difference(data1, data2)
data['QUOTES'] = sorted(data['QUOTES'], key=lambda entry: int(entry['pp']))
data['NOTES'] = sorted(data['NOTES'], key=lambda entry: int(entry['pp']))
#dump
return data
#main allows unix piping
if __name__ == '__main__':
if len(sys.argv) < 2:
sys.exit('No input file... Aborting.')
json.dump(run(sys.argv[1], sys.stdin), sys.stdout)
json.dump(data, sys.stdout)

18
parse/compare.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
INDXFILE='.indx'
for i in *.mmd; do
f=$i
done
if [[ ! -f $INDXFILE ]]; then
make --quiet index
elif [[ $f -nt $INDXFILE ]]; then
rm $INDXFILE
make --quiet index
fi
#parse-skim.sh | compare.py .indx
parse-plfr.sh | compare.py .indx

View File

@ -1,9 +1,4 @@
#!/usr/bin/env python
# reads (1) a formatted md file, (2) json formatted index
# and injects (sorts and appends) in the md file the information
# contained in the index
from statemachine import StateMachine
import sys, os, string, json, shutil, codecs, traceback
@ -19,6 +14,7 @@ def error(c):
sys.stderr.write('Unidentifiable line:\n'+ l)
def eof(c):
print "eof"
return
def parse(c):
@ -124,13 +120,10 @@ def emit_quotes(list):
def emit_quote(data):
emit_line("<!--page " + data['pp'] + "-->\n\n")
emit_line(">\"" + data['quote'] + "\" p." + data['pp'] + "\n")
emit_line(">\"" + data['quote'] + "\" pp." + data['pp'] + "\n")
emit_line('\n')
def emit_remaining_notes():
print "emit_remaining_notes"
rest = []
for i in notes:
rest.extend(notes[i])
@ -175,27 +168,25 @@ def backupfile(p):
shutil.copy2(p, bak)
return codecs.open(bak, 'r', 'utf-8')
def run(filename1, filename2):
global fileout, fileref, notes, quotes, notes_cnt
if __name__ == '__main__':
if len(sys.argv) < 2:
sys.exit('No input file... Aborting.')
# fp1 should be the incoming .mmd file
try:
fileref = backupfile(filename1)
fileout = open_fileoutput(filename1)
fileref = backupfile(sys.argv[1])
fileout = open_fileoutput(sys.argv[1])
fileout.seek(0)
except:
sys.exit("Can't open file " + filename1 + ". Aborting.")
sys.exit("Can't open file " + sys.argv[1] + ". Aborting.")
if len(sys.argv) < 3:
fp2 = sys.stdin
else:
try:
fp2 = open_file(filename2)
fp2 = open_file(sys.argv[2])
except:
# it may be stdin
try:
fp2 = filename2
fp2.tell()
except:
sys.exit("Can't open file " + filename2 + ". Aborting.")
sys.exit("Can't open file " + sys.argv[2] + ". Aborting.")
# fp2 should be the incoming (json) data to inject in fp1
@ -220,6 +211,8 @@ def run(filename1, filename2):
quotes = reoder(data['QUOTES'])
notes = reoder(data['NOTES'])
print quotes
notes_cnt = 0
try:
@ -241,23 +234,8 @@ def run(filename1, filename2):
finally:
fileout.close()
fileref.close()
return trace
#main allows unix piping
if __name__ == '__main__':
if len(sys.argv) < 2:
sys.exit('No input file... Aborting.')
if len(sys.argv) < 3:
fp2 = sys.stdin
else:
fp2 = sys.argv[2]
trace = run(sys.argv[1], fp2)
sys.exit(trace)

16
parse/inject.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash
for i in *.mmd; do
f=$i
done
if [[ -z '$f' ]]; then
echo "No mmd file (markdown source file) in directory... Aborting.";
exit;
fi
# inject new nnnotes in source file
compare.sh | inject.py $f
# update index
make --quiet index

13
parse/parse-plfr.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
for i in *.pdf; do
f=$i
done
if [[ -d '$f' ]]; then
echo "$f No pdf file in directory... Aborting.";
exit;
fi
plfr -json $f

77
parse/parse-skim.py Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/python
from statemachine import StateMachine
import sys, string, json
output = {'QUOTES' : [], 'NOTES' : []}
def error(c):
fp, l = c
sys.stderr.write('Unidentifiable line:\n'+ l)
def eof(c):
print json.dumps(output, sys.stdout)
def parse(c):
fp, l = c
while 1:
line = fp.readline()
if not line: return eof, (fp, line)
if line[:1] == '*': return section(line), (fp, line)
else: continue
def highlight(c):
fp, l = c
p = page(l)
text = fp.readline()
output['QUOTES'].append({'pp' : p, 'quote' : text.strip()})
return parse(c)
def anchored_note(c):
fp, l = c
p = page(l)
text = fp.readline()
fp.readline()
note = fp.readline()
output['QUOTES'].append({'pp' : p, 'quote' : text.strip()})
output['NOTES'].append({'pp' : p, 'note' : note.strip()})
return parse(c)
def box(c):
fp, l = c
p = page(l)
text = fp.readline()
output['QUOTES'].append({'pp' : p, 'quote' : text.strip()})
return parse(c)
def text_note(c):
fp, l = c
p = page(l)
text = fp.readline()
output['NOTES'].append({'pp' : p, 'note' : text.strip()})
return parse(c)
## helper fncts
def section(line):
line = string.upper(line)
if string.find(line, 'HIGHLIGHT') >= 0: return highlight
elif string.find(line, 'ANCHORED NOTE') >= 0: return anchored_note
elif string.find(line, 'BOX') >= 0: return box
elif string.find(line, 'TEXT NOTE') >= 0: return text_note
else: return parse
def page(line):
return line.rstrip('\n').split(',')[-1].split(' ')[-1]
if __name__ == '__main__':
m = StateMachine();
m.add_state(parse)
m.add_state(highlight)
m.add_state(anchored_note)
m.add_state(box)
m.add_state(text_note)
m.add_state(error, end_state=1)
m.add_state(eof, end_state=1)
m.set_start(parse)
m.run((sys.stdin, ''))

23
parse/parse-skim.sh Executable file
View File

@ -0,0 +1,23 @@
#!/bin/bash
for i in *.pdf; do
f=$i
done
if [[ -d '$f' ]]; then
echo "$f No pdf file in directory... Aborting.";
exit;
fi
filename="${f%.*}".txt
skimnotes get -format txt $f
if [[ ! -f $filename ]]; then
echo "No skim notes in pdf $f. Aborting.";
exit;
fi
parse-skim.py < $filename
rm $filename

View File

@ -1,8 +1,5 @@
#!/usr/bin/env python
# parses the information containned in a formatted md file
# and constructs a json formatted index
from statemachine import StateMachine
import sys, string, re, json
@ -20,14 +17,14 @@ def is_tag_identifier(line):
markups = {'QUOTES' : (is_quote_identifier, 'pp', 'tags', 'quote', 'fpc'), 'NOTES' : (is_note_identifier, '#', 'tags', 'note', 'fpc')}
output = {'QUOTES' : [], 'NOTES' : []}
fpindex = None
def error(c):
fp, l = c
sys.stderr.write('Unidentifiable line:\n'+ l)
def eof(c):
json.dump(output, fpindex)
fpindx = open('.indx','wb')
json.dump(output, fpindx)
def parse(c):
fp, l = c
@ -126,16 +123,12 @@ def escape_quote(line):
if(not line.strip().startswith('>')):
return line
l = re.sub('\"*\"', '', line.strip()[1:])
return re.sub('p.[0-9]+', '', l)
return re.sub('pp.[0-9]+', '', l)
def escape_note(line):
return re.sub('^[0-9]+.', '', line).strip()
def run(fpin, fpout):
global fpindex
fpindex = fpout
if __name__ == '__main__':
m = StateMachine();
m.add_state(parse)
m.add_state(NOTES)
@ -144,9 +137,4 @@ def run(fpin, fpout):
m.add_state(error, end_state=1)
m.add_state(eof, end_state=1)
m.set_start(parse)
m.run((fpin, ''))
#main allows unix piping
if __name__ == '__main__':
fpindx = open('.indx','wb')
run(sys.stdin, fpindx)
m.run((sys.stdin, ''))

63
parse/refactorbib.py Executable file
View File

@ -0,0 +1,63 @@
#!/usr/bin/env python
import sys, os, json
from optparse import OptionParser
def open_file(p):
if not os.path.exists(p):
sys.exit('File %s does not exists... Aborting.' % p)
return open(p, 'rb')
def refactor(data):
data_out = {}
for d in data:
nid = ''
if(d['author']):
nid = d['author'][0]['family'] + d['issued']['date-parts'][0][0]
elif(d['editor']):
nid = d['editor'][0]['family'] + d['issued']['date-parts'][0][0]
if nid in data_out:
for c in range(97, 122):
nid = nid + chr(c)
if not nid in data:
break
data_out[nid] = d
return data_out
if __name__ == '__main__':
p = OptionParser();
p.add_option('-i', '--index', action="store_true", help="prints out index")
options, args = p.parse_args()
if len(args) < 1:
sys.exit('No input file... Aborting.')
try:
fp = open_file(args[0])
except:
sys.exit("Can't open file " + args[0] + ". Aborting.")
try:
data = json.loads(fp.read())
except:
e = "<refactorbib> Error loading data from" + sys.argv[1] + ". Aborting.\n"
if sdata:
e += "Traceback: " + sdata1
sys.exit(e)
finally:
fp.close()
out = refactor(data)
if options.index:
for e in out.keys():
print '> ' + e + ' - ' + out[e]['title'] + ' '
else:
print json.dumps(out, sort_keys=True, indent=2, separators=(',', ': '))

59
rrref.sh Executable file
View File

@ -0,0 +1,59 @@
#!/bin/bash
WHERE=$( cd $(dirname $0) ; pwd -P )
# default configs
if [[ -e "$WHERE/CONFIG" ]]; then
source $WHERE/CONFIG
fi
OUT=txt # defaults to txt output
OPTIND=1 #reset getopts
while getopts :b:r:o: opts; do
case $opts in
b)
BIB=$OPTARG
;;
r)
REF=$OPTARG
;;
o)
OUT=$OPTARG
;;
?)
echo "invalid option -$OPTARG";
exit;
;;
:)
echo "option -$OPTARG requires an argument";
exit;
;;
esac
done
shift $((OPTIND-1))
# bibliographe's path set?
if [[ -z "$BIBLIOGRAPHE_PATH" ]]; then
echo "No bibliographe";
exit;
fi
# bibliography exists?
if [[ -z "$BIB" ]]; then
echo "No bibliography"
exit;
fi
# referebce exists?
if [[ -z "$REF" ]]; then
echo "No reference"
exit;
fi
./parse/refactorbib.py $BIB > tmpbib.json
REFERENCE=$(node $BIBLIOGRAPHE_PATH/generatebib.js --data tmpbib.json --items [\"$REF\"] --output $OUT)
echo $REFERENCE
rm tmpbib.json

View File

@ -1,26 +0,0 @@
#!/usr/bin/env python
import os, sys
from setuptools import setup, find_packages
PACKAGE = 'nnnotes'
with open('README') as file:
README = file.read()
setup(
name = 'nnnotes',
version = 'v0',
packages = find_packages(),
package_data = {PACKAGE: ['template/Makefile', 'template/*.mmd', 'template/p/*']},
scripts = ['bin/nnnote', 'bin/iiindex', 'bin/iiinject'],
provides=[PACKAGE],
author = 'gauthiier',
author_email = 'd@gauthiier.info',
url = 'https://github.com/gauthiier/nnnotes',
long_description=README,
classifiers=[
"Topic :: Utilities",
"License :: MIT License",
]
)

View File

@ -16,25 +16,22 @@ $(OUT)%.pdf : %.mmd
#html
HTML := $(patsubst %.mmd,$(OUT)%.html,$(wildcard *.mmd))
$(OUT)%.html: %.mmd
$(OUT)%.html : %.mmd
pandoc $< -s -t html5 --template=p/nnnote-template.html5 --filter pandoc-citeproc --csl $(CSL) --bibliography $(BIB) -o $@
all: dir $(HTML) index
all : dir $(HTML) index
index:
iiindex
index :
parse.py < $(wildcard *.mmd)
inject:
iiinject
clean:
clean :
rm -f $(HTML)
rm -f $(PDF)
neat:
rm *.bak
rebuild: clean all
rebuild : clean all
dir:
dir :
mkdir -p $(OUT)