From d2ea3a6c19a3729b830bf2a3069b47dfc9b54e7b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 20 Oct 2017 23:23:16 +0200 Subject: [PATCH] case-insensitive search for non-ASCII --- cps/db.py | 62 +++++++++++++++++++++------------------ cps/web.py | 41 ++++++++++++++------------ optional-requirements.txt | 2 +- 3 files changed, 57 insertions(+), 48 deletions(-) diff --git a/cps/db.py b/cps/db.py index f337ea4c..8be77977 100755 --- a/cps/db.py +++ b/cps/db.py @@ -26,43 +26,45 @@ def title_sort(title): title = title.replace(prep, '') + ', ' + prep return title.strip() +def lowercase(string): + return string.lower() Base = declarative_base() books_authors_link = Table('books_authors_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('author', Integer, ForeignKey('authors.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('author', Integer, ForeignKey('authors.id'), primary_key = True) ) books_tags_link = Table('books_tags_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('tag', Integer, ForeignKey('tags.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('tag', Integer, ForeignKey('tags.id'), primary_key = True) ) books_series_link = Table('books_series_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('series', Integer, ForeignKey('series.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('series', Integer, ForeignKey('series.id'), primary_key = True) ) books_ratings_link = Table('books_ratings_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('rating', Integer, ForeignKey('ratings.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('rating', Integer, ForeignKey('ratings.id'), primary_key = True) ) books_languages_link = Table('books_languages_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('lang_code', Integer, ForeignKey('languages.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('lang_code', Integer, ForeignKey('languages.id'), primary_key = True) ) books_publishers_link = Table('books_publishers_link', Base.metadata, - Column('book', Integer, ForeignKey('books.id'), primary_key=True), - Column('publisher', Integer, ForeignKey('publishers.id'), primary_key=True) + Column('book', Integer, ForeignKey('books.id'), primary_key = True), + Column('publisher', Integer, ForeignKey('publishers.id'), primary_key = True) ) class Identifiers(Base): __tablename__ = 'identifiers' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) type = Column(String) val = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -110,7 +112,7 @@ class Identifiers(Base): class Comments(Base): __tablename__ = 'comments' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) text = Column(String) book = Column(Integer, ForeignKey('books.id')) @@ -125,8 +127,8 @@ class Comments(Base): class Tags(Base): __tablename__ = 'tags' - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(String) + id = Column(Integer, primary_key = True, autoincrement = True) + name = Column(String(collation = 'NOCASE')) def __init__(self, name): self.name = name @@ -138,8 +140,8 @@ class Tags(Base): class Authors(Base): __tablename__ = 'authors' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) link = Column(String) @@ -155,8 +157,8 @@ class Authors(Base): class Series(Base): __tablename__ = 'series' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) def __init__(self, name, sort): @@ -170,7 +172,7 @@ class Series(Base): class Ratings(Base): __tablename__ = 'ratings' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) rating = Column(Integer) def __init__(self, rating): @@ -183,7 +185,7 @@ class Ratings(Base): class Languages(Base): __tablename__ = 'languages' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) lang_code = Column(String) def __init__(self, lang_code): @@ -195,8 +197,8 @@ class Languages(Base): class Publishers(Base): __tablename__ = 'publishers' - id = Column(Integer, primary_key=True) - name = Column(String) + id = Column(Integer, primary_key = True) + name = Column(String(collation = 'U_NOCASE')) sort = Column(String) def __init__(self, name,sort): @@ -211,7 +213,7 @@ class Publishers(Base): class Data(Base): __tablename__ = 'data' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) book = Column(Integer, ForeignKey('books.id')) format = Column(String) uncompressed_size = Column(Integer) @@ -232,8 +234,8 @@ class Books(Base): DEFAULT_PUBDATE = "0101-01-01 00:00:00+00:00" - id = Column(Integer, primary_key=True) - title = Column(String) + id = Column(Integer, primary_key = True) + title = Column(String(collation = 'U_NOCASE')) sort = Column(String) author_sort = Column(String) timestamp = Column(String) @@ -275,7 +277,7 @@ class Books(Base): class Custom_Columns(Base): __tablename__ = 'custom_columns' - id = Column(Integer, primary_key=True) + id = Column(Integer, primary_key = True) label = Column(String) name = Column(String) datatype = Column(String) @@ -294,6 +296,7 @@ def setup_db(): global engine global session global cc_classes + global conn if config.config_calibre_dir is None or config.config_calibre_dir == u'': content = ub.session.query(ub.Settings).first() @@ -319,6 +322,7 @@ def setup_db(): ub.session.commit() config.loadSettings() conn.connection.create_function('title_sort', 1, title_sort) + conn.connection.create_function('lower', 1, lowercase) if not cc_classes: cc = conn.execute("SELECT id, datatype FROM custom_columns") @@ -365,6 +369,6 @@ def setup_db(): # Base.metadata.create_all(engine) Session = sessionmaker() - Session.configure(bind=engine) + Session.configure(bind=conn) session = Session() return True diff --git a/cps/web.py b/cps/web.py index ef44328d..ff822b3e 100755 --- a/cps/web.py +++ b/cps/web.py @@ -659,11 +659,11 @@ def feed_normal_search(): def feed_search(term): if term: - entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.like("%" + term + "%")), - db.Books.series.any(db.Series.name.like("%" + term + "%")), - db.Books.authors.any(db.Authors.name.like("%" + term + "%")), - db.Books.publishers.any(db.Publishers.name.like("%" + term + "%")), - db.Books.title.like("%" + term + "%")))\ + entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), + db.Books.series.any(db.Series.name.ilike("%" + term + "%")), + db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), + db.Books.publishers.any(db.Publishers.name.ilike("%" + term + "%")), + db.Books.title.ilike("%" + term + "%")))\ .filter(common_filters()).all() entriescount = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entriescount, entriescount) @@ -909,7 +909,7 @@ def get_authors_json(): if request.method == "GET": query = request.args.get('q') # entries = db.session.execute("select name from authors where name like '%" + query + "%'") - entries = db.session.query(db.Authors).filter(db.Authors.name.like("%" + query + "%")).all() + entries = db.session.query(db.Authors).filter(db.Authors.name.ilike("%" + query + "%")).all() json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -920,7 +920,7 @@ def get_tags_json(): if request.method == "GET": query = request.args.get('q') # entries = db.session.execute("select name from tags where name like '%" + query + "%'") - entries = db.session.query(db.Tags).filter(db.Tags.name.like("%" + query + "%")).all() + entries = db.session.query(db.Tags).filter(db.Tags.iname.ilike("%" + query + "%")).all() # for x in entries: # alfa = dict(name=x.name) json_dumps = json.dumps([dict(name=r.name) for r in entries]) @@ -998,7 +998,7 @@ def get_languages_json(): def get_series_json(): if request.method == "GET": query = request.args.get('q') - entries = db.session.query(db.Series).filter(db.Series.name.like("%" + query + "%")).all() + entries = db.session.query(db.Series).filter(db.Series.name.ilike("%" + query + "%")).all() # entries = db.session.execute("select name from series where name like '%" + query + "%'") json_dumps = json.dumps([dict(name=r.name) for r in entries]) return json_dumps @@ -1014,8 +1014,8 @@ def get_matching_tags(): title_input = request.args.get('book_title') include_tag_inputs = request.args.getlist('include_tag') exclude_tag_inputs = request.args.getlist('exclude_tag') - q = q.filter(db.Books.authors.any(db.Authors.name.like("%" + author_input + "%")), - db.Books.title.like("%" + title_input + "%")) + q = q.filter(db.Books.authors.any(db.Authors.name.ilike("%" + author_input + "%")), + db.Books.title.ilike("%" + title_input + "%")) if len(include_tag_inputs) > 0: for tag in include_tag_inputs: q = q.filter(db.Books.tags.any(db.Tags.id == tag)) @@ -1577,13 +1577,18 @@ def update(): @login_required_if_no_ano def search(): term = request.args.get("query").strip() + if term: - entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.like("%" + term + "%")), - db.Books.series.any(db.Series.name.like("%" + term + "%")), - db.Books.authors.any(db.Authors.name.like("%" + term + "%")), - db.Books.publishers.any(db.Publishers.name.like("%" + term + "%")), - db.Books.title.like("%" + term + "%")))\ + entries = db.session.query(db.Books).filter(db.or_(db.Books.tags.any(db.Tags.name.ilike("%" + term + "%")), + db.Books.series.any(db.Series.name.ilike("%" + term + "%")), + db.Books.authors.any(db.Authors.name.ilike("%" + term + "%")), + db.Books.publishers.any(db.Publishers.name.ilike("%" + term + "%")), + db.Books.title.ilike("%" + term + "%")))\ .filter(common_filters()).all() +# entries = db.session.query(db.Books).with_entities(db.Books.title).filter(db.Books.title.ilike("%" + term + "%")).all() + #result = db.session.execute("select name from authors where lower(name) like '%" + term.lower() + "%'") + #entries = result.fetchall() + #result.close() return render_title_template('search.html', searchterm=term, entries=entries) else: return render_title_template('search.html', searchterm="") @@ -1625,9 +1630,9 @@ def advanced_search(): lang.name = _(isoLanguages.get(part3=lang.lang_code).name) searchterm.extend(language.name for language in language_names) searchterm = " + ".join(filter(None, searchterm)) - q = q.filter(db.Books.authors.any(db.Authors.name.like("%" + author_name + "%")), - db.Books.title.like("%" + book_title + "%"), - db.Books.publishers.any(db.Publishers.name.like("%" + publisher + "%"))) + q = q.filter(db.Books.authors.any(db.Authors.name.ilike("%" + author_name + "%")), + db.Books.title.ilike("%" + book_title + "%"), + db.Books.publishers.any(db.Publishers.name.ilike("%" + publisher + "%"))) for tag in include_tag_inputs: q = q.filter(db.Books.tags.any(db.Tags.id == tag)) for tag in exclude_tag_inputs: diff --git a/optional-requirements.txt b/optional-requirements.txt index cf743dbb..7a2ad451 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -12,4 +12,4 @@ rsa==3.4.2 six==1.10.0 uritemplate==3.0.0 goodreads>=0.3.2 -python-Levenshtein>=0.12.0 +#python-Levenshtein>=0.12.0