peisongxiao/coin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated gui.py to pull text from images

Browse Source
This commit is contained in:
Peisong Xiao
2024-03-14 23:04:36 -04:00
parent 0a830b2dd7
commit 42ce7aa21e
6 changed files with 266 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,2 +1,4 @@
downloaded_files/
testing/
cropped.png
backup/

8
coin.org
View File

@ -15,3 +15,11 @@ head against seemingly random Walmart abbreviations for items!) using
a scraper
Finally, it stores your shopping trip in a database.
* Dependencies
+ Python
+ PyQt6
+ pytesseract
+ OpenCV (On Python)
+ PIL
+ Selenium (undetected)

128
coin.ui Normal file
View File

@ -0,0 +1,128 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MainWindow</class>
<widget class="QMainWindow" name="MainWindow">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>1280</width>
<height>720</height>
</rect>
</property>
<property name="font">
<font>
<family>Droid Sans Mono</family>
<pointsize>12</pointsize>
</font>
</property>
<property name="windowTitle">
<string>coin</string>
</property>
<property name="autoFillBackground">
<bool>true</bool>
</property>
<property name="tabShape">
<enum>QTabWidget::Rounded</enum>
</property>
<widget class="QWidget" name="centralwidget">
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QWidget" name="widget" native="true">
<property name="sizePolicy">
<sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="minimumSize">
<size>
<width>200</width>
<height>0</height>
</size>
</property>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QScrollArea" name="imageView">
<property name="verticalScrollBarPolicy">
<enum>Qt::ScrollBarAlwaysOn</enum>
</property>
<property name="horizontalScrollBarPolicy">
<enum>Qt::ScrollBarAlwaysOn</enum>
</property>
<property name="widgetResizable">
<bool>true</bool>
</property>
<widget class="QWidget" name="scrollAreaWidgetContents">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>594</width>
<height>644</height>
</rect>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QLabel" name="photo">
<property name="text">
<string>Photo</string>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
</property>
</widget>
</item>
</layout>
</widget>
</widget>
</item>
</layout>
</widget>
</item>
<item>
<widget class="QWidget" name="widget_2" native="true">
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QWidget" name="widget_3" native="true">
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QPushButton" name="openButton">
<property name="text">
<string>Open image</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="getNameButton">
<property name="text">
<string>Get item names</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="saveButton">
<property name="text">
<string>Save to database</string>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="models"/>
</item>
</layout>
</widget>
</item>
<item>
<widget class="QTableView" name="itemsTable"/>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
<widget class="QStatusBar" name="statusbar"/>
</widget>
<resources/>
<connections/>
</ui>

139
gui.py
View File

@ -2,32 +2,66 @@
import pytesseract
import cv2, os, sys
from PIL import Image
import PyQt5
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5 import uic
from PyQt5 import QtCore, QtGui, QtWidgets
import PyQt6
from PyQt6.QtGui import *
from PyQt6.QtCore import *
from PyQt6.QtWidgets import *
from PyQt6 import uic
from PyQt6 import QtCore, QtGui, QtWidgets
import glob
# [!!!] Change this if tesseract can't find any data
models_path = '/usr/share/tessdata/'
# [!!!] Change this if tesseract or coin can't find any data
MODELS_PATH = '/usr/share/tessdata/'
# [!!!] Sane values for scaling
SCALE_MIN = 0.3
SCALE_MAX = 4.0
# How much (in decimal) to change the scale for every degree
SCALE_DELTA = 8 * 360 * 1
# [!!!] Sane values for recognition
WIDTH_MIN = 10
HEIGHT_MIN = 10
# Automatically finds all available language models
models_list = glob.glob(models_path + "*.traineddata")
models_list = glob.glob(MODELS_PATH + "*.traineddata")
model_names = []
for path in models_list:
base_name = os.path.basename(path)
base_name = os.path.splitext(base_name)[0]
model_names.append(base_name)
if len(model_names) == 0:
print("No tesseract models found at", MODELS_PATH, "!", file=sys.stderr)
sys.exit(0)
class coin_app(QtWidgets.QMainWindow):
def __init__(self):
QtWidgets.QMainWindow.__init__(self)
self.ui = uic.loadUi('selector.ui', self)
self.ui = uic.loadUi('coin.ui', self)
self.image = None
self.ui.openButton.clicked.connect(self.open)
self.rubberBand = QRubberBand(QRubberBand.Shape.Rectangle, self)
self.ui.photo.setMouseTracking(True)
self.ui.photo.installEventFilter(self)
self.ui.photo.setAlignment(PyQt6.QtCore.Qt.AlignmentFlag.AlignTop)
self.model = model_names[0]
print("Model selected as:", self.model)
self.models.addItems(model_names)
self.models.currentTextChanged.connect(self.update_now)
self.models.setCurrentIndex(model_names.index(self.model))
self.items = QStandardItemModel()
self.itemsTable.setModel(self.items)
self.items.setColumnCount(1)
self.scale = 1.0
def update_now(self, value):
self.model = value
print("Model selected as:", self.model)
def open(self):
filename = QFileDialog.getOpenFileName(self, 'Select File')
@ -35,16 +69,91 @@ class coin_app(QtWidgets.QMainWindow):
self.image = cv2.imread(str(filename[0]))
if self.image.size == 0:
print("Error: image is empty!")
print("Error: image is empty!", file=sys.stderr)
frame = cv2.cvtColor(self.image, cv2.COLOR_BGR2RGB)
image = QImage(frame, frame.shape[1], frame.shape[0],
frame.strides[0], QImage.Format_RGB888)
frame.strides[0], QImage.Format.Format_RGB888)
self.ui.photo.setPixmap(QPixmap.fromImage(image))
print("Updated photo")
self.pixmap = QPixmap.fromImage(image)
self.ui.photo.setPixmap(self.pixmap)
self.scale = 1.0
print("Updated photo!")
print("Photo scale set to:", self.scale)
def scale_update(self):
self.ui.photo.setPixmap(self.pixmap.scaled(
int(self.scale * self.pixmap.width()),
int(self.scale * self.pixmap.height())))
print("Photo scale set to:", self.scale)
def image_to_text(self, cropped_image):
gray = cv2.cvtColor(cropped_image, cv2.COLOR_BGR2GRAY)
gray = cv2.medianBlur(gray, 1)
crop = Image.fromarray(gray)
text = pytesseract.image_to_string(crop, lang=self.model)
print("Text selected:", text)
return text
def eventFilter(self, source, event):
# Handling the selection box
if event.type() == QEvent.Type.MouseButtonPress and source is self.ui.photo:
self.org = self.mapFromGlobal(event.globalPosition())
self.top_left = event.position()
self.rubberBand.setGeometry(QRect(self.org.toPoint(), QSize()))
self.rubberBand.show()
return True
elif event.type() == QEvent.Type.MouseMove and source is self.ui.photo:
if self.rubberBand.isVisible():
pos = self.mapFromGlobal(event.globalPosition()).toPoint()
pos = QPoint(int(max(pos.x(), 0)),
int(max(pos.y(), 0)))
self.rubberBand.setGeometry(
QRect(self.org.toPoint(),
pos).normalized())
return True
elif event.type() == QEvent.Type.MouseButtonRelease and source is self.ui.photo:
pos = event.position()
self.top_left = QPoint(int(max(min(pos.x(), self.top_left.x()), 0)),
int(max(min(pos.y(), self.top_left.y()), 0)))
if self.rubberBand.isVisible():
self.rubberBand.hide()
rect = self.rubberBand.geometry()
self.x1 = int(self.top_left.x() / self.scale)
self.y1 = int(self.top_left.y() / self.scale)
width = rect.width() / self.scale
height = rect.height() / self.scale
self.x2 = int(self.x1 + width)
self.y2 = int(self.y1 + height)
else:
return False
if width >= WIDTH_MIN and height >= HEIGHT_MIN and self.image is not None:
print("Cropped image:", self.x1, self.y1, self.x2, self.y2)
self.crop = self.image[self.y1:self.y2, self.x1:self.x2]
cv2.imwrite("cropped.png", self.crop)
self.text = self.image_to_text(self.crop)
##self.ui.textEdit()
return True
# Resizing the photo
elif event.type() == QEvent.Type.Wheel and self.image is not None and source is self.ui.photo:
modifiers = QApplication.keyboardModifiers()
if modifiers == Qt.KeyboardModifier.ControlModifier:
self.scale = self.scale + event.angleDelta().y() / SCALE_DELTA
if self.scale < SCALE_MIN:
self.scale = SCALE_MIN
if self.scale > SCALE_MAX:
self.scale = SCALE_MAX
self.scale_update()
return True
elif event.type() == QEvent.Type.MouseButtonDblClick and source is self.ui.photo:
self.open()
return True
return False
app = QtWidgets.QApplication(sys.argv)
mainWindow = coin_app()
mainWindow.show()
sys.exit(app.exec_())
sys.exit(app.exec())

124
selector.ui
View File

@ -1,124 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MainWindow</class>
<widget class="QMainWindow" name="MainWindow">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>1280</width>
<height>720</height>
</rect>
</property>
<property name="font">
<font>
<family>Droid Sans Mono</family>
<pointsize>12</pointsize>
</font>
</property>
<property name="windowTitle">
<string>coin</string>
</property>
<property name="autoFillBackground">
<bool>true</bool>
</property>
<property name="tabShape">
<enum>QTabWidget::Rounded</enum>
</property>
<widget class="QWidget" name="centralwidget">
<widget class="QPushButton" name="openButton">
<property name="geometry">
<rect>
<x>680</x>
<y>20</y>
<width>121</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Open image</string>
</property>
</widget>
<widget class="QScrollArea" name="imageView">
<property name="geometry">
<rect>
<x>20</x>
<y>20</y>
<width>651</width>
<height>671</height>
</rect>
</property>
<property name="verticalScrollBarPolicy">
<enum>Qt::ScrollBarAlwaysOn</enum>
</property>
<property name="horizontalScrollBarPolicy">
<enum>Qt::ScrollBarAlwaysOn</enum>
</property>
<property name="widgetResizable">
<bool>true</bool>
</property>
<widget class="QWidget" name="scrollAreaWidgetContents">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>635</width>
<height>655</height>
</rect>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0">
<widget class="QLabel" name="photo">
<property name="text">
<string>Photo</string>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
</property>
</widget>
</item>
</layout>
</widget>
</widget>
<widget class="QPushButton" name="saveButton">
<property name="geometry">
<rect>
<x>970</x>
<y>20</y>
<width>171</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Save to database</string>
</property>
</widget>
<widget class="QPushButton" name="getNameButton">
<property name="geometry">
<rect>
<x>810</x>
<y>20</y>
<width>151</width>
<height>31</height>
</rect>
</property>
<property name="text">
<string>Get item names</string>
</property>
</widget>
<widget class="QTableView" name="itemsTable">
<property name="geometry">
<rect>
<x>685</x>
<y>61</y>
<width>581</width>
<height>631</height>
</rect>
</property>
</widget>
</widget>
<widget class="QStatusBar" name="statusbar"/>
</widget>
<resources/>
<connections/>
</ui>

4
upc.py
View File

@ -4,7 +4,7 @@ from seleniumbase import Driver
driver = Driver(uc=True)
driver.implicitly_wait(5)
def get_name_from_upc(upc: str) -> str:
def get_name_from_upc(upc: str, wait_interval=10) -> str:
url = "https://stocktrack.ca/wm/index.php?s=wm&upc=" + upc
driver.get(url)
@ -39,7 +39,7 @@ def get_name_from_upc(upc: str) -> str:
print("Iteration No. ", times)
times = times + 1
time.sleep(20)
time.sleep(wait_interval)
page = str(driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML"))
t = page.find(str_t)