Working on multiprocessing

Gui test/globvar.py

@@ -6,9 +6,17 @@
 # for direct usage in assembly programming using nasm/yasm.
 
 from pathlib import Path
+import multiprocessing
 
 def init():
+    global home
     home = str(Path.home())
+    global num_cores
+    num_cores = multiprocessing.cpu_count()
+    global queue
+    queue = 0
+    global fraction
+    fraction = 0
     global worker
     worker = []
     global listener

Gui test/h2inc.glade

@@ -38,6 +38,7 @@
   <object class="GtkApplicationWindow" id="window">
     <property name="can_focus">False</property>
     <property name="title" translatable="yes">h2inc</property>
+    <property name="resizable">False</property>
     <child>
       <object class="GtkBox" id="box1">
         <property name="visible">True</property>

Gui test/h2inc.glade~

@@ -37,8 +37,7 @@
   </object>
   <object class="GtkApplicationWindow" id="window">
     <property name="can_focus">False</property>
-    <property name="title" translatable="yes">H2INC</property>
+    <property name="title" translatable="yes">h2inc</property>
-    <property name="hide_titlebar_when_maximized">True</property>
     <child>
       <object class="GtkBox" id="box1">
         <property name="visible">True</property>

Gui test/h2inc_gtk.py

@@ -11,7 +11,6 @@ import sys
 import gi
 gi.require_version('Gtk', '3.0')
 from gi.repository import Gtk, Gio, GObject as gobject
-#import time
 import globvar
 from h2inc_mp import start_workers
 from h2inc_fp import sourcedir_filecnt, sourcedir_foldercnt
@@ -42,7 +41,7 @@ class H2INC:
         self.obj = builder.get_object
         self.obj("window").set_application(app)
         self.obj("window").set_wmclass("h2inc","h2inc")
-        self.obj("window").set_title("h2inc -  v.0.0.1")
+        self.obj("window").set_title("h2inc -  v.0.1.1")
         self.obj("window").show_all()
         self.obj("source_entry").set_text(globvar.defsrc)
         self.obj("destination_entry").set_text(globvar.defdest)
@@ -61,8 +60,9 @@ class H2INC:
         self.app.run(argv)
 
     def callbackDisplay(self, obj, fraction, text, index, data=None):
-        self.obj("progress_label").set_text("{} of {}".format(index+1, self.filecnt))
+        self.new_value = self.obj("progressbar").get_fraction()+fraction
-        self.obj("progressbar").set_fraction(fraction)
+        self.obj("progress_label").set_text("{} of {}".format(index+1, globvar.filecnt))
+        self.obj("progressbar").set_fraction(self.new_value)
 
     def callbackFinished(self, obj, data=None):
         if self.process==None:
@@ -120,6 +120,7 @@ class H2INC:
         elif response == Gtk.ResponseType.CANCEL:
             print("Cancel")
 
+        globvar.fraction = 1/globvar.filecnt
         dialog.destroy()
 
     def on_destination_button_clicked(self,widget):
@@ -180,6 +181,8 @@ class H2INC:
                     print(globvar.foldercnt)
                     self.obj("numfolders_label").set_text(str(globvar.foldercnt))
 
+        globvar.fraction = 1/globvar.filecnt
+        
     def on_translate_button_clicked(self, widget, data=None):
         self.obj("sourceframe").set_sensitive(False)
         self.obj("translate_button").set_sensitive(False)

Gui test/h2inc_gtk_old.py

@@ -7,25 +7,167 @@
 # Program to convert C-header (*.h) files to nasm include files (*.inc),
 # for direct usage in assembly programming using nasm/yasm.
 
+import os
 import sys
+import io
 import gi
 gi.require_version('Gtk', '3.0')
 from gi.repository import Gtk, Gio, GObject as gobject
-#import time
+import time
-import globvar
+from multiprocessing import Process, Value, Lock, Pool, Queue
-from h2inc_mp import start_workers
+import threading
-from h2inc_fp import sourcedir_filecnt, sourcedir_foldercnt
 
-globvar.init()
+gobject.threads_init()
 
-class H2INC:
+class Listener(gobject.GObject):
+    __gsignals__ = {
+        'updated' : (gobject.SIGNAL_RUN_LAST,
+                     gobject.TYPE_NONE,
+                     (gobject.TYPE_FLOAT, gobject.TYPE_STRING, gobject.TYPE_INT)),
+        'finished': (gobject.SIGNAL_RUN_LAST,
+                     gobject.TYPE_NONE,
+                     ())
+    }
+
+    def __init__(self, queue):
+        gobject.GObject.__init__(self)
+        self.queue = queue
+
+    def go(self):
+        print("Listener has started")
+        while True:
+            # Listen for results on the queue and process them accordingly                            
+            data = self.queue.get()
+            # Check if finished                                                                       
+            if data[1]=="finished":
+                print("Listener is finishing.")
+                self.emit("finished")
+                return
+            else:
+                self.emit('updated', data[0], data[1], data[2])
+
+gobject.type_register(Listener)
+
+class Worker():
+    def __init__(self, queue, filecnt, filelist):
+        self.queue = queue
+        self.filecnt = filecnt
+
+    def go(self):
+        print("The worker has started doing some work (counting from 0 to 9)")
+        for i in range(self.filecnt):
+            proportion = (float(i+1))/self.filecnt
+            self.queue.put((proportion, "working...", i))
+            time.sleep(0.01)
+            process_file(filelist[i])
+        self.queue.put((1.0, "finished"))
+        print("The worker has finished.")
+
+tupline = []
+preproc = ()
+filelist = []
+folderlist = []
+cnt = 0
+srcdir = ''
+destdir = ''
+#num_cores = multiprocessing.cpu_count()
+fileindex = 0
+filecnt = 0
+incinc = ''
+defdir = False
+defsrc = 'usr/include'
+defdest = '~'
+count = 0
+
+def sourcedir_filecnt(sourcedir):
+    ### Return the number of files, ending with '.h', in sourcedir - including subdirectories ###
+    cnt = 0
+    global filelist
+    global srcdir
+    srcdir = sourcedir
+    for folderName, subfolders, files in os.walk(sourcedir):
+        for file in files:
+            if file.lower().endswith('.h'):
+                cnt += 1
+                filelist += [folderName + '/' + file]
+                print(folderName + '/' + file)
+    # print(filelist)
+    return cnt
+
+
+def sourcedir_foldercnt(sourcedir):
+    ### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
+    global cnt
+    global folderlist
+    for folderName, subfolders, files in os.walk(sourcedir):
+        if subfolders:
+            for subfolder in subfolders:
+                sourcedir_foldercnt(subfolder)
+        tempf = [file for file in files if file.lower().endswith('.h')]
+        if tempf:
+            cnt = cnt + 1
+            # print(folderName)
+            folderlist += [folderName]
+    # print(folderlist)
+    # print(len(folderlist))
+    return cnt
+
+def process_file(data):
+    global count
+    outfile = ''
+    inputfile = data
+    encodings = ['utf-8', 'latin-1', 'windows-1250', 'windows-1252', 'ascii',
+                 'big5', 'big5hkscs', 'cp037', 'cp273', 'cp424', 'cp437', 'cp500',
+                 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', 'cp855', 'cp856',
+                 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', 'cp863', 'cp864', 'cp865',
+                 'cp866', 'cp869', 'cp874', 'cp875', 'cp932', 'cp949', 'cp950', 'cp1006',
+                 'cp1026', 'cp1125', 'cp1140', 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254',
+                 'cp1255', 'cp1256', 'cp1257', 'cp1258', 'cp65001', 'euc-jp', 'euc-jis-2004',
+                 'euc-jisx0213', 'euc-kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022-jp',
+                 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', 'iso2022-jp-3',
+                 'iso2022-jp-ext', 'iso2022-kr', 'iso8859-2', 'iso8859-3', 'iso8859-4',
+                 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10',
+                 'iso8859-11', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', 'johab',
+                 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'mac-cyrillic', 'mac-greek',
+                 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', 'ptcp154',
+                 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', 'utf-32', 'utf-32-be',
+                 'utf-32-le', 'utf-16', 'utf-16-be', 'utf-16-le', 'utf-7', 'utf-8-sig']
+    for e in encodings:
+        try:
+            fh = io.open(data, 'r', encoding=e)
+            fh.readlines()
+            fh.seek(0)
+        except UnicodeDecodeError:
+            print('got unicode error with %s , trying different encoding' % e)
+        else:
+            # print('opening the file with encoding:  %s ' % e)
+            break
+            # print(os.path.basename(data))
+    for lines in fh:
+        outfile = outfile + lines
+    fh.close()
+    outputfile = os.path.splitext(inputfile)[0] + '.inc'
+    outputfile = str(outputfile).replace(srcdir, destdir)
+    count += 1
+    print(str(count)+'->'+outputfile)
+    if not os.path.exists(os.path.dirname(outputfile)):
+        try:
+            os.makedirs(os.path.dirname(outputfile))
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != FileExistsError:
+                raise
+    newfile = open(outputfile, "w")
+    newfile.write(outfile)
+    newfile.close()
+
+class ExampleApp:
 
     global app
+    global destlabel
 
     def __init__(self):
 
         self.filecnt = 0
-        self.foldercnt = 0
         self.fileindex = 0
         self.process = None
         self.app = Gtk.Application.new("org.h2inc", Gio.ApplicationFlags(0))
@@ -41,13 +183,12 @@ class H2INC:
 
         self.obj = builder.get_object
         self.obj("window").set_application(app)
-        self.obj("window").set_wmclass("h2inc","h2inc")
+        self.obj("window").set_wmclass("h2inc_gtk","h2inc_gtk")
-        self.obj("window").set_title("h2inc -  v.0.0.1")
         self.obj("window").show_all()
-        self.obj("source_entry").set_text(globvar.defsrc)
-        self.obj("destination_entry").set_text(globvar.defdest)
         self.obj("default_dir_checkbutton").set_active(False)
+        self.obj("default_dir_checkbutton").emit("toggled")
         self.obj("include_checkbutton").set_active(True)
+        self.obj("include_checkbutton").emit("toggled")
 
         button = Gtk.Button.new_from_stock(Gtk.STOCK_CANCEL)
         button.set_property("can-default",True)
@@ -95,34 +236,38 @@ class H2INC:
         self.on_dialog_close(widget)
 
     def on_source_button_clicked(self,widget):
+
         dialog = Gtk.FileChooserDialog("Select source directory!",
                                     self.obj("window"),
                                     Gtk.FileChooserAction.SELECT_FOLDER,
                                     (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                                     Gtk.STOCK_APPLY, Gtk.ResponseType.OK))
         dialog.set_default_size(600, 300)
-        Gtk.FileChooser.set_filename(dialog, globvar.srcdir)
+        Gtk.FileChooser.set_filename(dialog, srcdir)
 
         response = dialog.run()
         if response == Gtk.ResponseType.OK:
-            globvar.filecnt = sourcedir_filecnt(dialog.get_filename())
+            self.filecnt = sourcedir_filecnt(dialog.get_filename())
-            if globvar.filecnt >0:
+            if self.filecnt >0:
-                print(globvar.filecnt)
+                print(self.filecnt)
                 self.obj("source_entry").set_text(dialog.get_filename())
                 self.obj("destination_label").set_sensitive(True)
                 self.obj("destination_entry").set_sensitive(True)
                 self.obj("destination_button").set_sensitive(True)
                 self.obj("numfiles_label").set_text(str(self.filecnt))
-                self.obj("progress_label").set_text("{} of {}".format(globvar.fileindex, globvar.filecnt))
+                self.obj("progress_label").set_text("{} of {}".format(fileindex, self.filecnt))
-                globvar.foldercnt = sourcedir_foldercnt(dialog.get_filename())
+                foldercnt = sourcedir_foldercnt(dialog.get_filename())
-                if globvar.foldercnt >0:
+                if foldercnt >0:
-                    self.obj("numfolders_label").set_text(str(globvar.foldercnt))
+                    self.obj("numfolders_label").set_text(str(foldercnt))
         elif response == Gtk.ResponseType.CANCEL:
             print("Cancel")
 
         dialog.destroy()
 
     def on_destination_button_clicked(self,widget):
+        global destdir
+        global incinc
+
         dialog = Gtk.FileChooserDialog("Select destination directory!",
                                     self.obj("window"),
                                     Gtk.FileChooserAction.SELECT_FOLDER,
@@ -132,59 +277,82 @@ class H2INC:
 
         response = dialog.run()
         if response == Gtk.ResponseType.OK:
-            globvar.destdir = dialog.get_filename()
+            destdir = dialog.get_filename()
             self.obj("destination_entry").set_text(dialog.get_filename())
             self.obj("include_checkbutton").set_sensitive(True)
             self.obj("translation_frame").set_sensitive(True)
-            print(globvar.srcdir)
+            print(srcdir)
             if self.obj("include_checkbutton").get_active() == True:
-                globvar.incinc = '/include'
+                incinc = '/include'
-                globvar.destdir = globvar.destdir+globvar.incinc
+                destdir = destdir+incinc
-            print(globvar.destdir)
+            print(destdir)
         elif response == Gtk.ResponseType.CANCEL:
             print("Cancel")
 
         dialog.destroy()
 
     def on_include_checkbutton_toggled(self, widget):
+        global destdir
+        global incinc
         if self.obj("include_checkbutton").get_active() == True:
-            if globvar.destdir.lower().endswith('/include'):
+            incinc = '/include'
-                globvar.destdir = globvar.destdir
+            destdir = destdir+incinc
-            else:
-                globvar.destdir = globvar.destdir+'/include'
         if self.obj("include_checkbutton").get_active() == False:
-            #globvar.incinc = ''
+            incinc = ''
-            globvar.destdir = globvar.destdir.replace('/include', '')
+            destdir = destdir.replace('/include', '')
-        self.obj("destination_entry").set_text(globvar.destdir)
+        self.obj("destination_entry").set_text(destdir)
-        print(globvar.destdir)
+        print(destdir)
 
     def on_default_dir_checkbutton_toggled(self, widget):
-        globvar.defdir = self.obj("default_dir_checkbutton").get_active()
+        global defdir
-        if globvar.defdir == True:
+        global srcdir
-            globvar.srcdir = globvar.defsrc
+        global destdir
-            self.obj("source_entry").set_text(globvar.srcdir)
+        global incinc
-            globvar.destdir = globvar.defdest
+        defdir = self.obj("default_dir_checkbutton").get_active()
-            self.obj("destination_entry").set_text(globvar.destdir)
+        if defdir == True:
+            srcdir = defsrc
+            self.obj("source_entry").set_text(srcdir)
+            destdir = defdest+incinc
+            self.obj("destination_entry").set_text(destdir)
             self.obj("include_checkbutton").set_sensitive(True)
             self.obj("source_label").set_sensitive(False)
             self.obj("source_entry").set_sensitive(False)
             self.obj("source_button").set_sensitive(False)
             self.obj("translation_frame").set_sensitive(True)
-            globvar.filecnt = sourcedir_filecnt(globvar.srcdir)
+            filecnt = sourcedir_filecnt(defsrc)
-            if globvar.filecnt >0:
+            if filecnt >0:
-                print(globvar.filecnt)
+                print(filecnt)
-                self.obj("numfiles_label").set_text(str(globvar.filecnt))
+                foldercnt = sourcedir_foldercnt(defsrc)
-                self.obj("progress_label").set_text("{} of {}".format(globvar.fileindex, self.filecnt))
+                if foldercnt >0:
-                globvar.foldercnt = sourcedir_foldercnt(globvar.srcdir)
+                    self.obj("numfolders_label").set_text(str(foldercnt))
-                if globvar.foldercnt >0:
-                    print(globvar.foldercnt)
-                    self.obj("numfolders_label").set_text(str(globvar.foldercnt))
 
     def on_translate_button_clicked(self, widget, data=None):
         self.obj("sourceframe").set_sensitive(False)
         self.obj("translate_button").set_sensitive(False)
         
-        start_workers()
+        if self.process!=None:
+            return
 
-app = H2INC()
+        print("Creating shared Queue")
-app.run(sys.argv)
+        queue = Queue()
+
+        print("Creating Worker")
+        worker = Worker(queue, self.filecnt, filelist)
+
+        print("Creating Listener")
+        listener = Listener(queue)
+        listener.connect("updated",self.callbackDisplay)
+        listener.connect("finished",self.callbackFinished)
+
+        print("Starting Listener")
+        thread = threading.Thread(target=listener.go, args=())
+        thread.start()
+
+        print("Starting Worker")
+        self.process = Process(target=worker.go, args=())
+        self.process.start()
+
+        
+
+app = ExampleApp()
+app.run(sys.argv)

Gui test/h2inc_mp.py

@@ -11,10 +11,9 @@ from threading import Thread
 import threading
 import globvar
 import h2inc_fp
+import time
 
-num_cores = multiprocessing.cpu_count()
 lock = threading.Lock()
-queue = 0
 
 def process_queue():
     print("Worker {} working...".format(threading.current_thread().name))
@@ -26,19 +25,19 @@ def process_queue():
         #self.queue.put((1.0, "finished"))
         #print("The worker has finished.")
     while True:
-        cfile = queue.get()
+        cfile = globvar.queue.get()
         print(cfile)
         h2inc_fp.process_file(cfile)
-        queue.task_done()
+        globvar.queue.task_done()
 
 def start_workers():
-    global queue
     print("Creating shared Queue")
-    queue = Queue()
+    ts = time.time()
+    globvar.queue = Queue()
 
-    print("Number of cores:", num_cores)
+    print("Number of cores:", globvar.num_cores)
         
-    for n in range(num_cores):
+    for n in range(globvar.num_cores):
         print("Creating Worker", n)
 
         worker = threading.Thread(target=process_queue)
@@ -47,8 +46,10 @@ def start_workers():
 
     for cfile in globvar.filelist:
         print("Queueing {}".format(cfile))
-        queue.put(cfile)
+        globvar.queue.put(cfile)
 
-    queue.join()
+    globvar.queue.join()
+    te = time.time()-ts
+    print("Processing time: ", te)
     return