Lerking/h2inc-old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ee

Browse Source
This commit is contained in:
Jan Lerking
2018-07-29 14:14:04 +02:00
parent 00871f743f
commit 99b43200f4
7 changed files with 118 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/settings.json vendored
View File

@@ -1,5 +1,5 @@
{
"python.pythonPath": "/usr/bin/python3.5",
"python.pythonPath": "/home/jan/anaconda3/bin/python",
"python.linting.pylintEnabled": true,
"python.linting.pep8Enabled": false,
"python.linting.enabled": true

20
Multiprocessing test/ticker.glade
View File

@@ -3,7 +3,9 @@
<interface>
<requires lib="gtk+" version="3.12"/>
<object class="GtkApplicationWindow" id="window">
<property name="width_request">300</property>
<property name="can_focus">False</property>
<property name="opacity">0.97999999999999998</property>
<property name="resizable">False</property>
<child>
<object class="GtkBox" id="box1">
@@ -63,7 +65,7 @@
<object class="GtkLabel" id="ticker_put_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="valign">baseline</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items put in Queue:</property>
@@ -80,7 +82,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -106,7 +107,7 @@
<object class="GtkLabel" id="ticker_items_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items currently in Queue:</property>
<property name="justify">right</property>
@@ -121,7 +122,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -147,7 +147,7 @@
<object class="GtkLabel" id="ticker_processed_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items processed in Queue:</property>
<property name="justify">right</property>
@@ -162,7 +162,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -237,7 +236,7 @@
<object class="GtkLabel" id="work_put_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items put in Queue:</property>
<property name="justify">right</property>
@@ -253,7 +252,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -280,7 +278,7 @@
<object class="GtkLabel" id="work_items_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items currently in Queue:</property>
</object>
@@ -294,7 +292,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -320,7 +317,7 @@
<object class="GtkLabel" id="work_processed_label">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">end</property>
<property name="halign">start</property>
<property name="xpad">5</property>
<property name="label" translatable="yes">Items processed in Queue:</property>
</object>
@@ -334,7 +331,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>

7
Multiprocessing test/ticker.glade~
View File

@@ -58,6 +58,7 @@
<object class="GtkGrid" id="grid2">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="hexpand">True</property>
<child>
<object class="GtkLabel" id="ticker_put_label">
<property name="visible">True</property>
@@ -79,7 +80,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -120,7 +120,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -161,7 +160,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -252,7 +250,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -293,7 +290,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>
@@ -333,7 +329,6 @@
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="halign">start</property>
<property name="hexpand">True</property>
</object>
<packing>
<property name="left_attach">1</property>

82
Multiprocessing test/ticker_gtk.py
View File

@@ -25,18 +25,20 @@ gobject.threads_init()
class Listener(gobject.GObject):
__gsignals__ = {
'ticker_update' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
(gobject.TYPE_INT,)),
'handlers_update' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
(gobject.TYPE_FLOAT,
gobject.TYPE_FLOAT,
gobject.TYPE_FLOAT)),
gobject.TYPE_FLOAT,)),
'handlers_finished' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
()),
'workers_update' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
(gobject.TYPE_STRING,
gobject.TYPE_PYOBJECT)),
(gobject.TYPE_PYOBJECT,)),
'workers_finished' : (gobject.SIGNAL_RUN_LAST,
gobject.TYPE_NONE,
())
@@ -57,6 +59,10 @@ class Listener(gobject.GObject):
print("Ticker is finishing.")
self.emit("ticker_finished", data[0])
return
elif data[0]=="ticker_update":
print("Ticker is running!")
self.emit("ticker_update")
return
elif data[0]=="handler_update":
print("Handlers are working.")
self.emit("handlers_update")
@@ -64,9 +70,10 @@ class Listener(gobject.GObject):
elif data[0]=="work_update":
print("Workers are working.")
self.emit("workers_update", data[1])
print("Signal: workers_update, ",data[1])
return
else:
self.emit('update_ticker', data[0], data[1], data[2])
return
gobject.type_register(Listener)
@@ -85,7 +92,9 @@ class Worker():
self.fractions.append(task)
print(task)
self.lock.acquire()
print("Lock acquired!")
try:
print("Getting qsize")
tasks = self.work_queue.qsize()
self.fractions.append(tasks)
print(tasks)
@@ -100,7 +109,10 @@ class Worker():
#tasks = tasks/1000
time.sleep(0.001)
self.work_queue.task_done()
self.lock.acquire()
self.signal_queue.put("work_update", self.fractions)
print(self.fractions)
self.lock.release()
self.work_queue.task_done()
self.signal_queue.put("work_finished")
@@ -121,6 +133,20 @@ class Handler():
self.ticker_queue.task_done()
self.signal_queue.put("")
class Ticker():
def __init__(self, sq, tq, i):
self.signal_queue = sq
self.ticker_queue = tq
self.tickers = i
self.iterations = 1
def go(self):
while self.iterations <= self.tickers:
self.ticker_queue.put(self.iterations)
time.sleep(0.01)
self.signal_queue.put("ticker_update", self.iterations)
self.iterations += 1
self.signal_queue.put("ticker_finished")
#def looprun(n):
#for i in tickers:
@@ -154,7 +180,7 @@ class Handler():
#for n in range(100):
#looprun(n+1)
class ticker:
class t_gtk:
global app
@@ -173,7 +199,7 @@ class ticker:
self.ticker_queue = multiprocessing.JoinableQueue()
self.work_queue = multiprocessing.JoinableQueue()
self.iterations = multiprocessing.Value('i', 0)
self.tickers = range(globvar.ticker_num)
self.tickers = globvar.ticker_num
self.work_num = 0
self.processes = []
self.threads = []
@@ -192,7 +218,7 @@ class ticker:
self.obj = builder.get_object
self.obj("window").set_application(app)
self.obj("window").set_wmclass("ticker","ticker")
self.obj("window").set_title("ticker - v.0.1.15")
self.obj("window").set_title("ticker - v.0.2.15")
self.obj("window").show_all()
self.obj("ticker_put_count_label").set_text(str(self.ticker_put_count))
self.obj("ticker_items_count_label").set_text(str(self.ticker_current_count))
@@ -207,27 +233,33 @@ class ticker:
def run(self, argv):
self.app.run(argv)
def update_ticker_put(self, index):
self.val = index*self.fraction
self.obj("ticker_put_count_label").set_text(str(index))
self.obj("ticker_put_progress").set_fraction(self.val)
def update_ticker(self, obj, index1, index2, index3, data=None):
self.val_1 = index1*self.fraction
self.val_2 = index2*self.fraction
self.val_3 = index3*self.fraction
self.obj("ticker_put_count_label").set_text(index1)
self.obj("ticker_put_progressbar").set_fraction(self.val_1)
self.obj("ticker_put_progress").set_fraction(self.val_1)
self.obj("ticker_items_count_label").set_text(index2)
self.obj("ticker_items_progressbar").set_fraction(self.val_2)
self.obj("ticker_items_progress").set_fraction(self.val_2)
self.obj("ticker_processed_count_label").set_text(index3)
self.obj("ticker_processed_progressbar").set_fraction(self.val_3)
self.obj("ticker_processed_progress").set_fraction(self.val_3)
def update_work(self, obj, index):
def update_work(self, obj, index, data=None):
print(index)
self.val_1 = index[0]*self.fraction
self.val_2 = index[1]*self.fraction
self.val_3 = index[2]*self.fraction
self.obj("work_put_count_label").set_text(index[0])
self.obj("work_put_progressbar").set_fraction(self.val_1)
self.obj("work_put_progress").set_fraction(self.val_1)
self.obj("work_items_count_label").set_text(index[1])
self.obj("work_items_progressbar").set_fraction(self.val_2)
self.obj("work_items_progress").set_fraction(self.val_2)
self.obj("work_processed_count_label").set_text(index[2])
self.obj("work_processed_progressbar").set_fraction(self.val_3)
self.obj("work_processed_progress").set_fraction(self.val_3)
def ticker_Finished(self, obj, data=None):
for t in range(20):
@@ -237,11 +269,11 @@ class ticker:
self.threads.append(t)
self.obj("ticker_put_count_label").set_text(self.ticker_num)
self.obj("ticker_put_progressbar").set_fraction(1.0)
self.obj("ticker_put_progress").set_fraction(1.0)
self.obj("ticker_items_count_label").set_text("0")
self.obj("ticker_items_progressbar").set_fraction(0.0)
self.obj("ticker_items_progress").set_fraction(0.0)
self.obj("ticker_processed_count_label").set_text(self.ticker_num)
self.obj("ticker_processed_progressbar").set_fraction(1.0)
self.obj("ticker_processed_progress").set_fraction(1.0)
def workers_Finished(self, obj, data=None):
if self.process==None:
@@ -251,11 +283,11 @@ class ticker:
self.process = None
self.obj("work_put_count_label").set_text(self.work_num)
self.obj("work_put_progressbar").set_fraction(1.0)
self.obj("work_put_progress").set_fraction(1.0)
self.obj("work_items_count_label").set_text("0")
self.obj("work_items_progressbar").set_fraction(0.0)
self.obj("work_items_progress").set_fraction(0.0)
self.obj("work_processed_count_label").set_text(self.work_num)
self.obj("work_processed_progressbar").set_fraction(1.0)
self.obj("work_processed_progress").set_fraction(1.0)
def on_window_destroy(self,window):
window.close()
@@ -263,6 +295,7 @@ class ticker:
def on_execute_clicked(self,widget):
print("Creating Listener")
listener = Listener(self.signal_queue)
listener.connect("ticker_update", self.update_ticker_put)
listener.connect("handlers_update",self.update_ticker)
listener.connect("handlers_finished",self.ticker_Finished)
listener.connect("workers_update",self.update_work)
@@ -273,8 +306,11 @@ class ticker:
thread.start()
print("Creating ticker queue")
for i in self.tickers:
self.ticker_queue.put(i)
t = Ticker(self.signal_queue, self.ticker_queue, self.tickers)
th = threading.Thread(target=t.go, args = ())
th.deamon = True
th.start()
#th.join()
print(self.ticker_queue.qsize())
for i in range(20):
@@ -297,7 +333,7 @@ class ticker:
app = ticker()
app = t_gtk()
app.run(sys.argv)

BIN
parser/__pycache__/keywords.cpython-35.pyc Normal file
View File

Binary file not shown.

BIN
parser/__pycache__/keywords.cpython-36.pyc Normal file
View File

Binary file not shown.

66
parser/parser.py
View File

@@ -2,20 +2,21 @@
# Python parser for c header files.
# Used for creating corresponding NASM include files.
import keywords
import os
import sys
import multiprocessing
import threading
import time
import keywords
keywords.init()
num_cores = multiprocessing.cpu_count()
work_queue = multiprocessing.JoinableQueue()
signal_queue = multiprocessing.JoinableQueue()
test_folder = "..\\gtk"
print(test_folder)
test_folder = "/usr/include"
filelist = []
folderlist = []
processes = []
threads = []
@@ -30,8 +31,16 @@ class Listener:
data = self.signal_queue.get()
print(data)
# Check if finished
if data[0]=="work_update":
if data[0] == "work_update":
print("Workers are working.")
self.signal_queue.task_done()
return
elif data[0] == "work_finished":
print("Workers are finished.")
self.signal_queue.task_done()
return
else:
self.signal_queue.task_done()
return
class Worker:
@@ -40,14 +49,17 @@ class Worker:
self.work_queue = wq
def go(self):
#while self.work_queue.get(2) != None:
while True:
print("Working")
#print("Working")
task = self.work_queue.get(2)
print(task)
time.sleep(0.001)
#print(task)
time.sleep(0.01)
self.work_queue.task_done()
print(self.work_queue.qsize())
self.signal_queue.put("work_update")
self.work_queue.task_done()
self.signal_queue.put("work_finished")
class parser:
def __init__(self, file):
@@ -59,9 +71,9 @@ class parser:
def get_token(self, keyword):
token = ""
if keyword in self.prep:
token = self.prep(keyword)
token = self.prep.values(keyword)
if keyword in self.reg:
token = self.reg(keyword)
token = self.reg.values(keyword)
return token
def parse_preprocess(self, token):
@@ -72,10 +84,10 @@ class parser:
token = {}
if word in self.prep:
token.keys = word
token.values = self.prep(word)
token.values = self.prep.values(word)
if word in self.reg:
token.keys = word
token.values = self.reg(word)
token.values = self.reg.values(word)
return token
# Creates a list of elements per line in file,
@@ -85,7 +97,7 @@ class parser:
word = [w for w in line.split()]
for w in word:
token = self.tokenize_word(w)
if token.value == 'PREPROCESS':
if token.values == 'PREPROCESS':
self.parse_preprocess(token)
return tupline
@@ -106,8 +118,9 @@ def sourcedir_filecnt(sourcedir):
return cnt
def sourcedir_foldercnt(sourcedir):
### Return the number of folders, if it contains '*.h' files, in sourcedir - including subdirectories ###
global cnt
# Return the number of folders, if it contains '*.h' files,
# in 'sourcedir' - including subdirectories.
cnt = 0
global folderlist
for folderName, subfolders, files in os.walk(sourcedir):
if subfolders:
@@ -127,7 +140,7 @@ def process_files(gui, source, dest):
global destdir
sourcedir = source
destdir = dest
pool = mp.Pool(processes=num_cores)
pool = multiprocessing.Pool(processes=num_cores)
pool.map(process_file, filelist)
def process_file(data):
@@ -176,10 +189,10 @@ def process_file(data):
newfile.write(outfile)
newfile.close()
test_folder = get_script_path()+'\\..\\gtk'
print(test_folder)
print('Number of *.h files in directory: ',sourcedir_filecnt(test_folder))
print(num_cores)
print('Number of processor cores: ',num_cores)
print("Creating Listener")
listener = Listener(signal_queue)
@@ -193,7 +206,7 @@ print(threads)
print("Creating work queue")
for i in filelist:
work_queue.put(i)
print(work_queue.qsize())
print(work_queue.qsize())
for i in range(num_cores):
w = Worker(signal_queue, work_queue)
@@ -203,6 +216,21 @@ for i in range(num_cores):
processes.append(p)
print(processes)
work_queue.join()
for i in range(num_cores):
work_queue.put(None)
print('Items on work queue: ',work_queue.qsize())
work_queue.join()
print('Items on signal queue: ',signal_queue.qsize())
signal_queue.join()
print("Closing down workers")
for p in processes:
print("Closing down:",p)
p.join()
print(processes)
print("Closing down Listener")
for t in threads:
t.join()
print(threads)