Compare commits

...

35 commits

Author SHA1 Message Date
Tejaswini Chile
2a045bf19c Merge branch 'feat/5913-search-improvements' of https://github.com/chatwoot/chatwoot into feat/5913-search-improvements 2022-12-26 13:45:22 +05:30
Tejaswini Chile
63fea6bd28 Merge branch 'feat/5913-search-improvements' of https://github.com/chatwoot/chatwoot into feat/5913-search-improvements 2022-12-26 13:44:43 +05:30
Tejaswini Chile
510b8aca30 Merge branch 'feat/5913-search-improvements' of https://github.com/chatwoot/chatwoot into feat/5913-search-improvements 2022-12-26 13:06:14 +05:30
Tejaswini Chile
6ea71008fd fix: added conversation created_at and first message in the text searched conversation data 2022-12-26 13:02:26 +05:30
Tejaswini Chile
ca8d156212 fix: added conversation created_at and first message in the text searched conversation data 2022-12-26 12:55:50 +05:30
Tejaswini Chile
9c0fb99dca spec fixes 2022-12-23 18:18:01 +05:30
Tejaswini Chile
ca7e912c04 codeclimate fix 2022-12-23 17:16:02 +05:30
Tejaswini Chile
5bb54794d8 fix: spec 2022-12-23 17:03:45 +05:30
Tejaswini Chile
32b95baeb2 fix: spec 2022-12-23 15:58:19 +05:30
Tejaswini Chile
3ec72ffce4
Merge branch 'develop' into feat/5913-search-improvements 2022-12-23 15:54:06 +05:30
Tejaswini Chile
08c75a5946 fix: agent role based search 2022-12-23 15:52:17 +05:30
Tejaswini Chile
91205627b4 fix: agent role based search 2022-12-23 15:43:01 +05:30
Tejaswini Chile
d9d5d087d3 fix: Account based rebuild index 2022-12-23 14:04:09 +05:30
Tejaswini Chile
c842c98769 fix: update in the text search query with includes 2022-12-21 22:53:25 +05:30
Tejaswini Chile
4f651d04ee fix: update in the text search query with includes 2022-12-21 19:07:05 +05:30
Tejaswini Chile
bec5b6e6ea fix: reducing JSON contents from search data 2022-12-21 13:30:25 +05:30
Tejaswini Chile
56488c6bae fix: specs and PR feedbacks 2022-12-21 13:30:25 +05:30
Tejaswini Chile
5bb4c12e6b fix: update inbox details for text search json 2022-12-21 13:30:25 +05:30
Tejaswini Chile
3447b56b18 fix: update inbox details for text search json 2022-12-21 13:30:25 +05:30
Tejaswini Chile
87eb798d7c fix: added the threshold for the multisearch 2022-12-21 13:30:25 +05:30
Tejaswini Chile
9681827776 fix: message with inbox and agent details 2022-12-21 13:30:25 +05:30
Tejaswini Chile
918eda22a6 fix: added json for agents details in conversation 2022-12-21 13:30:25 +05:30
Tejaswini Chile
a9b80da11c fix: Conversation to include contacts data 2022-12-21 13:30:25 +05:30
Tejaswini Chile
5c322e96a4 fix: Added pg_trgm extension 2022-12-21 13:30:25 +05:30
Tejaswini Chile
4d2afface2 fix: unwanted changes and the comments 2022-12-21 13:30:25 +05:30
Tejaswini Chile
a025271324 message json fix 2022-12-21 13:30:25 +05:30
Tejaswini Chile
604ca395ad fix: issue with current account not being present for some user 2022-12-21 13:30:25 +05:30
Tejaswini Chile
f513bdb97d fix: migration for rebuilding multi model search 2022-12-21 13:30:25 +05:30
Tejaswini Chile
723968f042 fix: specs 2022-12-21 13:30:25 +05:30
Tejaswini Chile
f3bea265f9 fix: JSON format 2022-12-21 13:30:25 +05:30
Tejaswini Chile
4eca37fd79 fix: JSON format 2022-12-21 13:30:25 +05:30
Tejaswini Chile
1a2349ae84 fix: new endpoint for the text search 2022-12-21 13:30:25 +05:30
Tejaswini Chile
c03a6602f3 fix: new endpoint for the text search 2022-12-21 13:30:25 +05:30
Tejaswini Chile
a8600d79f4 fix: search improvements for multiple model with separate results 2022-12-21 13:30:25 +05:30
Tejaswini Chile
23cd34cbd6 feat: Search improvements 2022-12-21 13:30:22 +05:30
20 changed files with 430 additions and 18 deletions

View file

@ -2,8 +2,8 @@ class Api::V1::Accounts::ConversationsController < Api::V1::Accounts::BaseContro
include Events::Types
include DateRangeHelper
before_action :conversation, except: [:index, :meta, :search, :create, :filter]
before_action :inbox, :contact, :contact_inbox, only: [:create]
before_action :conversation, except: [:index, :meta, :search, :create, :filter, :text_search]
before_action :inbox, :contact, :contact_inbox, :text_search, only: [:create]
def index
result = conversation_finder.perform
@ -11,6 +11,10 @@ class Api::V1::Accounts::ConversationsController < Api::V1::Accounts::BaseContro
@conversations_count = result[:count]
end
def text_search
@result = TextSearch.new(Current.user, params).perform
end
def meta
result = conversation_finder.perform
@conversations_count = result[:count]

View file

@ -0,0 +1,44 @@
class TextSearch
attr_reader :current_user, :current_account, :params
DEFAULT_STATUS = 'open'.freeze
def initialize(current_user, params)
@current_user = current_user
@current_account = @current_user.account || Current.account
@params = params
end
def perform
set_inboxes
{
messages: filter_messages,
conversations: filter_conversations,
contacts: filter_contacts
}
end
def set_inboxes
@inbox_ids = @current_user.assigned_inboxes.pluck(:id)
end
private
def filter_conversations
@conversations = PgSearch.multisearch((@params[:q]).to_s).where(
inbox_id: @inbox_ids, account_id: @current_account, searchable_type: 'Conversation'
).joins('INNER JOIN conversations ON pg_search_documents.searchable_id = conversations.id').includes(:searchable).limit(20).collect(&:searchable)
end
def filter_messages
@messages = PgSearch.multisearch((@params[:q]).to_s).where(
inbox_id: @inbox_ids, account_id: @current_account, searchable_type: 'Message'
).joins('INNER JOIN messages ON pg_search_documents.searchable_id = messages.id').includes(:searchable).limit(20).collect(&:searchable)
end
def filter_contacts
@contacts = PgSearch.multisearch((@params[:q]).to_s).where(
account_id: @current_account, searchable_type: 'Contact'
).joins('INNER JOIN contacts ON pg_search_documents.searchable_id = contacts.id').includes(:searchable).limit(20).collect(&:searchable)
end
end

View file

@ -0,0 +1,9 @@
class Conversations::AccountBasedSearchJob < ApplicationJob
queue_as :async_database_migration
def perform(account_id)
Contact.rebuild_pg_search_documents(account_id)
Conversation.rebuild_pg_search_documents(account_id)
Message.rebuild_pg_search_documents(account_id)
end
end

View file

@ -0,0 +1,9 @@
class Conversations::MultiSearchJob < ApplicationJob
queue_as :async_database_migration
def perform
Account.all.each do |account|
Conversations::AccountBasedSearchJob.perform_later(account.id)
end
end
end

View file

@ -0,0 +1,74 @@
module MultiSearchableHelpers
extend ActiveSupport::Concern
included do
PgSearch.multisearch_options = {
using: {
trigram: {
word_similarity: true,
threshold: 0.3
},
tsearch: { any_word: true }
}
}
def update_contact_search_document
return if contact_pg_search_record.present?
initialize_contact_pg_search_record.update!(
content: "#{contact.id} #{contact.email} #{contact.name} #{contact.phone_number} #{contact.account_id}",
conversation_id: id,
inbox_id: inbox_id
)
end
# NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
# We can not find conversation_id from contacts directly so we added this joins here.
def self.rebuild_pg_search_documents(account_id)
return unless self.name == 'Conversation'
rebuild_search_documents(account_id)
end
end
def rebuild_search_documents(account_id)
connection.execute <<~SQL.squish
INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
SELECT 'Conversation' AS searchable_type,
conversations.id AS searchable_id,
CONCAT_WS(' ', conversations.display_id, contacts.email, contacts.name, contacts.phone_number, conversations.account_id) AS content,
conversations.account_id::int AS account_id,
conversations.id::int AS conversation_id,
conversations.inbox_id::int AS inbox_id,
now() AS created_at,
now() AS updated_at
FROM conversations
INNER JOIN contacts
ON conversations.contact_id = contacts.id
WHERE conversations.account_id = #{account_id}
SQL
end
def contact_pg_search_record
contacts_pg_search_records.find_by(conversation_id: id, inbox_id: inbox_id)
end
def initialize_contact_pg_search_record
record = contacts_pg_search_records.find_by(conversation_id: nil, inbox_id: nil)
return record if record.present?
PgSearch::Document.new(
searchable_type: 'Contact',
searchable_id: contact_id,
account_id: account_id
)
end
def contacts_pg_search_records
PgSearch::Document.where(
searchable_type: 'Contact',
searchable_id: contact_id,
account_id: account_id
)
end
end

View file

@ -26,6 +26,13 @@ class Contact < ApplicationRecord
include Avatarable
include AvailabilityStatusable
include Labelable
include PgSearch::Model
include MultiSearchableHelpers
multisearchable(
against: [:id, :email, :name, :phone_number],
additional_attributes: ->(contact) { { conversation_id: nil, account_id: contact.account_id, inbox_id: nil } }
)
validates :account_id, presence: true
validates :email, allow_blank: true, uniqueness: { scope: [:account_id], case_sensitive: false },
@ -140,6 +147,28 @@ class Contact < ApplicationRecord
email_format
end
# NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
# We can not find conversation_id from contacts directly so we added this joins here.
def self.rebuild_pg_search_documents(account_id)
return super unless name == 'Contact'
connection.execute <<~SQL.squish
INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
SELECT 'Contact' AS searchable_type,
contacts.id AS searchable_id,
CONCAT_WS(' ', contacts.id, contacts.email, contacts.name, contacts.phone_number, contacts.account_id) AS content,
contacts.account_id::int AS account_id,
conversations.id::int AS conversation_id,
conversations.inbox_id::int AS inbox_id,
now() AS created_at,
now() AS updated_at
FROM contacts
INNER JOIN conversations
ON conversations.contact_id = contacts.id
WHERE contacts.account_id = #{account_id}
SQL
end
private
def ip_lookup

View file

@ -48,7 +48,15 @@ class Conversation < ApplicationRecord
include ActivityMessageHandler
include UrlHelper
include SortHandler
include PgSearch::Model
include MultiSearchableHelpers
multisearchable(
against: [:display_id, :name, :email, :phone_number, :account_id],
additional_attributes: lambda { |conversation|
{ conversation_id: conversation.id, account_id: conversation.account_id, inbox_id: conversation.inbox_id }
}
)
validates :account_id, presence: true
validates :inbox_id, presence: true
before_validation :validate_additional_attributes
@ -93,9 +101,11 @@ class Conversation < ApplicationRecord
after_update_commit :execute_after_update_commit_callbacks
after_create_commit :notify_conversation_creation
after_create_commit :update_contact_search_document, if: :contact_id?
after_commit :set_display_id, unless: :display_id?
delegate :auto_resolve_duration, to: :account
delegate :name, :email, :phone_number, to: :contact, allow_nil: true
def can_reply?
channel = inbox&.channel

View file

@ -33,6 +33,14 @@
class Message < ApplicationRecord
include MessageFilterHelpers
NUMBER_OF_PERMITTED_ATTACHMENTS = 15
include PgSearch::Model
include MultiSearchableHelpers
multisearchable(
against: [:content],
if: :allowed_message_types?,
additional_attributes: ->(message) { { conversation_id: message.conversation_id, account_id: message.account_id, inbox_id: message.inbox_id } }
)
before_validation :ensure_content_type
@ -162,6 +170,26 @@ class Message < ApplicationRecord
true
end
# NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
# We can not find conversation_id from contacts directly so we added this joins here.
def self.rebuild_pg_search_documents(account_id)
return super unless name == 'Message'
connection.execute <<~SQL.squish
INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
SELECT 'Message' AS searchable_type,
messages.id AS searchable_id,
CONCAT_WS(' ', messages.content) AS content,
messages.account_id::int AS account_id,
messages.conversation_id::int AS conversation_id,
messages.inbox_id::int AS inbox_id,
now() AS created_at,
now() AS updated_at
FROM messages
WHERE messages.account_id = #{account_id}
SQL
end
private
def ensure_content_type
@ -274,4 +302,8 @@ class Message < ApplicationRecord
conversation.update_columns(last_activity_at: created_at)
# rubocop:enable Rails/SkipsModelValidations
end
def allowed_message_types?
incoming? || outgoing?
end
end

View file

@ -0,0 +1,32 @@
json.payload do
json.conversations do
json.array! @result[:conversations] do |conversation|
json.id conversation.display_id
json.account_id conversation.account_id
json.created_at conversation.created_at.to_i
json.message do
json.partial! 'api/v1/models/multi_search_message', formats: [:json], message: conversation.messages.try(:first)
end
json.contact do
json.partial! 'api/v1/models/multi_search_contact', formats: [:json], contact: conversation.contact if conversation.try(:contact).present?
end
json.inbox do
json.partial! 'api/v1/models/multi_search_inbox', formats: [:json], inbox: conversation.inbox if conversation.try(:inbox).present?
end
json.agent do
json.partial! 'api/v1/models/multi_search_agent', formats: [:json], agent: conversation.assignee if conversation.try(:assignee).present?
end
end
end
json.contacts do
json.array! @result[:contacts] do |contact|
json.partial! 'api/v1/models/multi_search_contact', formats: [:json], contact: contact
end
end
json.messages do
json.array! @result[:messages] do |message|
json.partial! 'api/v1/models/multi_search_message', formats: [:json], message: message
end
end
end

View file

@ -2,7 +2,8 @@ json.id message.id
json.content message.content
json.inbox_id message.inbox_id
json.echo_id message.echo_id if message.echo_id
json.conversation_id message.conversation.display_id
# For deleted conversation, messages are not yet deleted [because of destroy_async] for this we added try block
json.conversation_id message.conversation.try(:display_id)
json.message_type message.message_type_before_type_cast
json.content_type message.content_type
json.status message.status

View file

@ -0,0 +1,5 @@
json.id agent.id
json.available_name agent.available_name
json.email agent.email
json.name agent.name
json.role agent.role

View file

@ -0,0 +1,5 @@
json.email contact.email
json.id contact.id
json.name contact.name
json.phone_number contact.phone_number
json.identifier contact.identifier

View file

@ -0,0 +1,4 @@
json.id inbox.id
json.channel_id inbox.channel_id
json.name inbox.name
json.channel_type inbox.channel_type

View file

@ -0,0 +1,15 @@
json.id message.id
json.content message.content
json.message_type message.message_type_before_type_cast
json.content_type message.content_type
json.source_id message.source_id
json.inbox_id message.inbox_id
json.created_at message.created_at.to_i
json.agent do
if message.conversation.try(:assignee).present?
json.partial! 'api/v1/models/multi_search_agent', formats: [:json], agent: message.conversation.try(:assignee)
end
end
json.inbox do
json.partial! 'api/v1/models/multi_search_inbox', formats: [:json], inbox: message.inbox if message.inbox.present? && message.try(:inbox).present?
end

View file

@ -71,6 +71,7 @@ Rails.application.routes.draw do
get :meta
get :search
post :filter
get :text_search
end
scope module: :conversations do
resources :messages, only: [:index, :create, :destroy]

View file

@ -12,20 +12,21 @@
# even put in dynamic logic, like a host-specific queue.
# http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
:queues:
- [low, 1]
- [scheduled_jobs, 1]
- [webhooks, 1]
- [bots, 1]
- [active_storage_analysis, 1]
- [action_mailbox_incineration, 1]
- [active_storage_purge, 1]
- [integrations, 2]
- [default, 2]
- [mailers, 2]
- [medium, 3]
- [events, 3]
- [action_mailbox_routing, 3]
- [high, 5]
- [async_database_migration, 1]
- [low, 2]
- [scheduled_jobs, 2]
- [webhooks, 2]
- [bots, 2]
- [active_storage_analysis, 2]
- [action_mailbox_incineration, 2]
- [active_storage_purge, 2]
- [integrations, 3]
- [default, 3]
- [mailers, 3]
- [medium, 4]
- [events, 4]
- [action_mailbox_routing, 4]
- [high, 6]
- [critical, 10]
# you can override concurrency based on environment

View file

@ -0,0 +1,22 @@
class CreatePgSearchDocuments < ActiveRecord::Migration[6.1]
def up
say_with_time('Creating table for pg_search multisearch') do
create_table :pg_search_documents do |t|
t.text :content
t.bigint 'conversation_id'
t.bigint 'account_id'
t.bigint 'inbox_id'
t.belongs_to :searchable, polymorphic: true, index: true
t.timestamps null: false
end
add_index :pg_search_documents, :account_id
add_index :pg_search_documents, :conversation_id
end
end
def down
say_with_time('Dropping table for pg_search multisearch') do
drop_table :pg_search_documents
end
end
end

View file

@ -0,0 +1,10 @@
class EnableMultiSearchable < ActiveRecord::Migration[6.1]
def up
::Conversations::MultiSearchJob.perform_now
execute 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
end
def down
PgSearch::Document.delete_all
end
end

View file

@ -14,6 +14,7 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_stat_statements"
enable_extension "pg_trgm"
enable_extension "pgcrypto"
enable_extension "plpgsql"
@ -399,7 +400,7 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
t.datetime "agent_last_seen_at"
t.jsonb "additional_attributes", default: {}
t.bigint "contact_inbox_id"
t.uuid "uuid", default: -> { "gen_random_uuid()" }, null: false
t.uuid "uuid", default: -> { "public.gen_random_uuid()" }, null: false
t.string "identifier"
t.datetime "last_activity_at", default: -> { "CURRENT_TIMESTAMP" }, null: false
t.bigint "team_id"
@ -674,6 +675,20 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
t.index ["user_id"], name: "index_notifications_on_user_id"
end
create_table "pg_search_documents", force: :cascade do |t|
t.text "content"
t.bigint "conversation_id"
t.bigint "account_id"
t.bigint "inbox_id"
t.string "searchable_type"
t.bigint "searchable_id"
t.datetime "created_at", precision: 6, null: false
t.datetime "updated_at", precision: 6, null: false
t.index ["account_id"], name: "index_pg_search_documents_on_account_id"
t.index ["conversation_id"], name: "index_pg_search_documents_on_conversation_id"
t.index ["searchable_type", "searchable_id"], name: "index_pg_search_documents_on_searchable"
end
create_table "platform_app_permissibles", force: :cascade do |t|
t.bigint "platform_app_id", null: false
t.string "permissible_type", null: false
@ -836,6 +851,9 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
t.jsonb "custom_attributes", default: {}
t.string "type"
t.text "message_signature"
t.datetime "locked_at"
t.integer "failed_attempts"
t.string "unlock_token"
t.index ["email"], name: "index_users_on_email"
t.index ["pubsub_token"], name: "index_users_on_pubsub_token", unique: true
t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true

View file

@ -0,0 +1,87 @@
require 'rails_helper'
describe ::TextSearch do
subject(:text_search) { described_class.new(user_1, params) }
let!(:account) { create(:account) }
let!(:user_1) { create(:user, account: account) }
let!(:user_2) { create(:user, account: account) }
let!(:inbox) { create(:inbox, account: account, enable_auto_assignment: false) }
before do
create(:inbox_member, user: user_1, inbox: inbox)
create(:inbox_member, user: user_2, inbox: inbox)
create(:contact, name: '1223', account_id: account.id)
create(:contact, name: 'Potter', account_id: account.id)
contact_2 = create(:contact, name: 'Harry Potter', account_id: account.id, email: 'harry@chatwoot.com')
conversation_1 = create(:conversation, account: account, inbox: inbox, assignee: user_1, display_id: 1213)
conversation_2 = create(:conversation, account: account, inbox: inbox, assignee: user_1, display_id: 1223)
create(:conversation, account: account, inbox: inbox, assignee: user_1, status: 'resolved', display_id: 13, contact_id: contact_2.id)
create(:conversation, account: account, inbox: inbox, assignee: user_2, display_id: 14)
create(:conversation, account: account, inbox: inbox, display_id: 15)
Current.account = account
create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'Ask Lisa')
create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'message_12')
create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'message_13')
create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'Pottery Barn order')
create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'message_22')
create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'message_23')
end
describe '#perform' do
context 'with text search' do
it 'filter conversations by number' do
params = { q: '122' }
result = described_class.new(user_1, params).perform
expect(result[:conversations].length).to eq 1
expect(result[:contacts].length).to eq 1
end
it 'filter message and contacts by string' do
params = { q: 'pot' }
result = described_class.new(user_1, params).perform
expect(result[:messages].length).to be 1
expect(result[:contacts].length).to be 2
end
it 'filter conversations by contact details' do
params = { q: 'pot' }
result = described_class.new(user_1, params).perform
expect(result[:conversations].length).to be 1
end
it 'filter conversations by contact email' do
params = { q: 'harry@chatwoot.com' }
result = described_class.new(user_1, params).perform
expect(result[:conversations].length).to be 1
end
end
context 'when create records in tables including multi search' do
let(:contact) { create(:contact, name: 'Welma', account_id: account.id, email: 'welma@scoobydoo.com') }
let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: user_1, status: 'open', contact_id: contact.id) }
it 'conversation creation pg search records' do
contact_search_record = PgSearch::Document.find_by(searchable_id: contact.id, searchable_type: contact.class.name)
conversation_search_record = PgSearch::Document.find_by(searchable_id: conversation.id, searchable_type: conversation.class.name)
expect(contact_search_record).to be_present
expect(conversation_search_record).to be_present
end
it 'conversation deletion deletes pg search records' do
contact.destroy!
conversation.destroy!
contact_search_record = PgSearch::Document.find_by(searchable_id: contact.id, searchable_type: contact.class.name)
conversation_search_record = PgSearch::Document.find_by(searchable_id: conversation.id, searchable_type: conversation.class.name)
expect(contact_search_record).to be_nil
expect(conversation_search_record).to be_nil
end
end
end
end