team4alfanous team mailing list archive
-
team4alfanous team
-
Mailing list archive
-
Message #00056
[Branch ~team4alfanous/alfanous/alfanous-git] Rev 239: * generator of unvocalized word => different vocalizations
------------------------------------------------------------
revno: 239
git commit: 0d8eed1b136e9c2bf6dfd1ef0c1c878f45f8168e
committer: Assem Chelli <assem.ch@xxxxxxxxx>
timestamp: Fri 2012-06-08 04:35:20 +0100
message:
* generator of unvocalized word => different vocalizations
modified:
Makefile
src/alfanous-import/Transformer.py
src/alfanous-import/main.py
--
lp:~team4alfanous/alfanous/alfanous-git
https://code.launchpad.net/~team4alfanous/alfanous/alfanous-git
Your team Alfanous team is subscribed to branch lp:~team4alfanous/alfanous/alfanous-git.
To unsubscribe from this branch go to https://code.launchpad.net/~team4alfanous/alfanous/alfanous-git/+edit-subscription
=== modified file 'Makefile'
--- Makefile 2012-04-15 18:18:57 +0000
+++ Makefile 2012-06-08 03:35:20 +0000
@@ -28,7 +28,8 @@
PREFIX?=/usr
CONFIG_INSTALL_PATH="$(DESTDIR)$(PREFIX)/share/alfanous-config/"
INDEX_INSTALL_PATH="$(DESTDIR)$(PREFIX)/share/alfanous-indexes/"
-WEB_INSTALL_PATH="$(DESTDIR)/var/www/alfanous-web/"
+WEB_INSTALL_PATH=$(DESTDIR)/var/www/alfanous-web/
+WEB_CGI_INSTALL_PATH=$(WEB_INSTALL_PATH)cgi
default:
@@ -101,6 +102,9 @@
transfer_derivations:
export PYTHONPATH=$(API_PATH) ; python $(QIMPORT) -t derivations $(DB_PATH)main.db $(DYNAMIC_RESOURCES_PATH)
+transfer_derivations:
+ export PYTHONPATH=$(API_PATH) ; python $(QIMPORT) -t vocalizations $(DB_PATH)main.db $(DYNAMIC_RESOURCES_PATH)
+
transfer_ara2eng_names:
export PYTHONPATH=$(API_PATH) ; python $(QIMPORT) -t ara2eng_names $(DB_PATH)main.db $(DYNAMIC_RESOURCES_PATH)
@@ -160,15 +164,14 @@
## build help
-help_all: help_api help_sphinx
+help_all: help_epydoc help_sphinx
help_epydoc:
- cd $(API_PATH); epydoc --html -v --graph all --no-sourcecode --show-imports -n alfanous -u alfanous.org . ; zip -9 alfanous-epydoc.zip ./html/* ; mv -f alfanous-epydoc.zip ../output ; rm -r ./html
+ mkdir -p output/$(VERSION);cd $(API_PATH); epydoc --html -v --graph all --no-sourcecode --show-imports -n alfanous -u alfanous.org . ; zip -9 alfanous-epydoc.zip ./html/* ; mv -f alfanous-epydoc.zip ../output/$(VERSION)/ ; rm -r ./html
help_sphinx:
- cd ./docs ; make dirhtml; zip -9 alfanous-sphinx-doc.zip ./_build/dirhtml/* ; mv -f alfanous-sphinx-doc.zip ../output ; rm -r ./_build/dirhtml
+ mkdir -p output/$(VERSION);cd ./docs ; make dirhtml; zip -9 alfanous-sphinx-doc.zip ./_build/dirhtml/* ; mv -f alfanous-sphinx-doc.zip ../output/$(VERSION)/ ; rm -r ./_build/dirhtml
-
## Qt forms ,dialogs and resources compilation
# PyQt is needed
# apt-get install pyqt4-dev-tools pyqt-tools
@@ -202,7 +205,7 @@
#wget ; mv to /localization/locale
## make packages
-dist_all: dist_egg dist_deb dist_rpm dist_sis dist_xpi dist_app
+dist_all: dist_egg_all dist_deb dist_rpm dist_sis dist_xpi dist_app
# generate all extentions and API's eggs
dist_egg_all: dist_egg_api dist_egg_pycorpus dist_egg_pyzekr dist_egg_qimport dist_egg_desktop
@@ -311,24 +314,45 @@
#test installation
alfanousDesktop &
-
-install_web: install_api install_index install_config
+## don't use it!!
+install_json: #install_api #install_index
+ cd $(API_PATH)alfanous-cgi ; mkdir -p $(WEB_CGI_INSTALL_PATH); cp -r alfanous-json.py $(WEB_CGI_INSTALL_PATH);
+ #cd ./interfaces/web/ ; cp htaccess $(WEB_INSTALL_PATH)".htaccess"
+ cd ./interfaces/web/ ; vi alfanous ; cp alfanous /etc/apache2/sites-available/ #configure well this file
+ chmod 755 -R $(WEB_CGI_INSTALL_PATH)
+ chmod +x $(WEB_INSTALL_PATH)cgi/alfanous_json.py
+
+ a2dissite alfanous
+ a2ensite alfanous
+ service apache2 reload
+ echo "127.0.0.1 alfanous.local" >> /etc/hosts ## must check existance first!!
+ xdg-open http://alfanous.local/ & ##launch default browser for test
+
+## don't use it!!
+install_json2: install_api install_index install_config
+ cd $(API_PATH)alfanous-cgi ; mkdir -p $(WEB_CGI_INSTALL_PATH); cp -r alfanous_json2.py $(WEB_CGI_INSTALL_PATH);
+ #cd ./interfaces/web/ ; cp htaccess $(WEB_INSTALL_PATH)".htaccess"
+ cd ./interfaces/web/ ; vi alfanous ; cp alfanous /etc/apache2/sites-available/ #configure well this file
+ chmod 755 -R $(WEB_INSTALL_PATH)
+ chmod +x $(WEB_INSTALL_PATH)cgi/alfanous_json2.py
+ sed -i 's/\"cgitb.enable\(\)\"/cgitb.enable\(\)/g' "$(WEB_INSTALL_PATH)cgi/alfanous_json2.py"
+ sed -i 's/\.\/indexes/\/usr\/share\/alfanous\-indexes/g' "$(WEB_INSTALL_PATH)cgi/alfanous_json2.py"
+ sed -i 's/\.\/configs/\/usr\/share\/alfanous\-config/g' "$(WEB_INSTALL_PATH)cgi/alfanous_json2.py"
+
+ cd $(WEB_INSTALL_PATH); mv -f \ cgi cgi; cd wui; sed -i 's/www\.alfanous\.org\/json/alfanous\.local\/cgi\-bin\/alfanous\_json\.py/g' index.*
+ a2dissite alfanous
+ a2ensite alfanous
+ service apache2 reload
+ echo "127.0.0.1 alfanous.local" >> /etc/hosts ## must check existance first!!
+ xdg-open http://alfanous.local/ & ##launch default browser for test
+
+## don't use it!!
+install_wui: #install_json
rm -r $(WEB_INSTALL_PATH)
mkdir -p $(WEB_INSTALL_PATH)
cd ./interfaces/web/ ; cp ./AGPL $(WEB_INSTALL_PATH)
cd ./interfaces/web/ ; cp -r wui $(WEB_INSTALL_PATH)
- cd $(API_PATH)alfanous-cgi ; mkdir "$(WEB_INSTALL_PATH)cgi"; cp -r *.py "$(WEB_INSTALL_PATH)cgi";
- #cd ./interfaces/web/ ; cp .htaccess "$(WEB_INSTALL_PATH)"
- cd ./interfaces/web/ ; cp alfanous /etc/apache2/sites-available/ #configure well this file
- chmod 755 $(WEB_INSTALL_PATH)
- chmod +x "$(WEB_INSTALL_PATH)cgi/alfanous-json.py"
- sed -i 's/\"cgitb.enable\(\)\"/cgitb.enable\(\)/g' "$(WEB_INSTALL_PATH)cgi/alfanous-json.py"
- sed -i 's/\.\/indexes/\/usr\/share\/alfanous\-indexes/g' "$(WEB_INSTALL_PATH)cgi/alfanous-json.py"
- sed -i 's/\.\/configs/\/usr\/share\/alfanous\-config/g' "$(WEB_INSTALL_PATH)cgi/alfanous-json.py"
-
- cd $(WEB_INSTALL_PATH); mv -f \ cgi cgi; cd wui; sed -i 's/www\.alfanous\.org\/json/alfanous\.local\/cgi\-bin\/alfanous\-json\.py/g' index.*
- a2dissite alfanous
- a2ensite alfanous
- service apache2 reload
- #echo "127.0.0.1 alfanous.local" >> /etc/hosts ## must check existance first!!
- #xdg-open http://alfanous.local/ & ##launch default browser for test
+ cd ./interfaces/web/ ; vi alfanous ; cp alfanous /etc/apache2/sites-available/ #configure well this file
+ chmod 755 -R $(WEB_INSTALL_PATH)
+ echo "127.0.0.1 alfanous.local" >> /etc/hosts ## must check existance first!!
+ xdg-open http://alfanous.local/ & ##launch default browser for test
=== modified file 'src/alfanous-import/Transformer.py'
--- src/alfanous-import/Transformer.py 2012-06-01 02:54:15 +0000
+++ src/alfanous-import/Transformer.py 2012-06-08 03:35:20 +0000
@@ -41,7 +41,7 @@
import sqlite3 as lite
-
+from alfanous.main import QuranicSearchEngine
from alfanous.Indexing import QseDocIndex
from alfanous.TextProcessing import QStandardAnalyzer, QDiacAnalyzer, QArabicSymbolsFilter, QUthmaniDiacAnalyzer, QUthmaniAnalyzer
from alfanous.Exceptions import Ta7rif
@@ -66,6 +66,7 @@
if not( os.path.exists( ixpath ) ):
os.makedirs( ixpath )
self.__storage = FileStorage( ixpath )
+ self.__ixpath = ixpath
self.__dypypath = dypypath
pass
@@ -374,7 +375,7 @@
return raw_str
def transfer_derivations( self ):
- """ load word props from database and save them as a list in a dynamic py """
+ """ load word derivations from database and save them as a list in a dynamic py """
cur = self.__mainDb.cursor()
levels = ["word_", "lemma", "root"]
cur.execute( "select " + ",".join( levels ) + " from word" )
@@ -394,14 +395,47 @@
return raw_str
+ def transfer_vocalizations( self ):
+ """ load indexed vocalized words from the main index and save them as a list in a dynamic py """
+ QSE = QuranicSearchEngine( self.__ixpath )
+ mfw = QSE.most_frequent_words( 9999999, "aya_" )
+
+ V = QArabicSymbolsFilter( \
+ shaping = False, \
+ tashkil = True, \
+ spellerrors = False, \
+ hamza = False \
+ ).normalize_all
+
+
+ vocalization_dict = {}
+ for w in mfw:
+ word = w[1]
+ if vocalization_dict.has_key( V( word ) ):
+ vocalization_dict[V( word )].append( word )
+ else:
+ vocalization_dict[V( word )] = [word]
+
+ raw_str = self.dheader + u"\nvocalization_dict=" + str( vocalization_dict ).replace( ",", ",\n" )
+
+ fich = open( self.__dypypath + "vocalizations_dyn.py", "w+" )
+ fich.write( raw_str )
+
+ return raw_str
+
+
+
+
+
if __name__ == "__main__":
- #T = Transformer( ixpath="../indexes/main/" , dypypath="dynamic_resources/",dbpath="../../resources/DB/main.db")
- T = Transformer( ixpath = "../../indexes/word/" , dypypath = "../alfanous/dynamic_resources/", dbpath = "../../resources/DB/main.db" )
+ T = Transformer( ixpath = "../../indexes/main/" , dypypath = "../alfanous/dynamic_resources/", dbpath = "../../resources/databases/main.db" )
+ #T = Transformer( ixpath = "../../indexes/word/" , dypypath = "../alfanous/dynamic_resources/", dbpath = "../../resources/DB/main.db" )
#T.transfer_stopwords()
#T.transfer_synonymes()
#T.transfer_word_props()
#T.transfer_derivations()
+ T.transfer_vocalizations()
#T.transfer_ara2eng_names()
#T.transfer_std2uth_words()
#T.make_spellerrors_dict()
@@ -412,7 +446,7 @@
#T.build_speller(indexname="AYA_SPELL", fields=["aya"])
#T.build_speller(indexname="SUBJECT_SPELL", fields=["subject"])
- wordqcSchema = T.build_schema( tablename = 'wordqc' )
+ #wordqcSchema = T.build_schema( tablename = 'wordqc' )
#T.build_docindex(wordqcSchema,tablename='wordqc')
#T.build_speller(indexname="WORD_SPELL", fields=["word"])
=== modified file 'src/alfanous-import/main.py'
--- src/alfanous-import/main.py 2012-06-01 02:54:15 +0000
+++ src/alfanous-import/main.py 2012-06-08 03:35:20 +0000
@@ -41,7 +41,7 @@
commands.add_option( "-x", "--index", dest = "index", type = "choice", choices = ["main", "extend" , "word"],
help = "create indexes", metavar = "TYPE" )
-commands.add_option( "-t", "--transfer", dest = "transfer", type = "choice", choices = ["stopwords", "synonyms", "word_props", "derivations", "ara2eng_names", "std2uth_words"],
+commands.add_option( "-t", "--transfer", dest = "transfer", type = "choice", choices = ["stopwords", "synonyms", "word_props", "derivations", "vocalizations", "ara2eng_names", "std2uth_words"],
help = "transfer from database to dynamic resources", metavar = "RESOURCE" )
commands.add_option( "-p", "--speller", dest = "speller", type = "choice", choices = ["aya", "subject", "word"],
@@ -105,6 +105,8 @@
T.transfer_word_props()
elif options.transfer == "derivations":
T.transfer_derivations()
+ elif options.transfer == "vocalizations":
+ T.transfer_vocalizations()
elif options.transfer == "ara2eng_names":
T.transfer_ara2eng_names()
elif options.transfer == "std2uth_words":
