Merge branch 'feat/5913-search-improvements' of https://github.com/chatwoot/chatwoot into feat/5913-search-improvements

app/controllers/api/v1/accounts/conversations_controller.rb

@@ -2,8 +2,8 @@ class Api::V1::Accounts::ConversationsController < Api::V1::Accounts::BaseContro
   include Events::Types
   include DateRangeHelper
 
-  before_action :conversation, except: [:index, :meta, :search, :create, :filter]
+  before_action :conversation, except: [:index, :meta, :search, :create, :filter, :text_search]
-  before_action :inbox, :contact, :contact_inbox, only: [:create]
+  before_action :inbox, :contact, :contact_inbox, :text_search, only: [:create]
 
   def index
     result = conversation_finder.perform
@@ -11,6 +11,10 @@ class Api::V1::Accounts::ConversationsController < Api::V1::Accounts::BaseContro
     @conversations_count = result[:count]
   end
 
+  def text_search
+    @result = TextSearch.new(Current.user, params).perform
+  end
+
   def meta
     result = conversation_finder.perform
     @conversations_count = result[:count]

app/finders/text_search.rb

@@ -0,0 +1,44 @@
+class TextSearch
+  attr_reader :current_user, :current_account, :params
+
+  DEFAULT_STATUS = 'open'.freeze
+
+  def initialize(current_user, params)
+    @current_user = current_user
+    @current_account = @current_user.account || Current.account
+    @params = params
+  end
+
+  def perform
+    set_inboxes
+    {
+      messages: filter_messages,
+      conversations: filter_conversations,
+      contacts: filter_contacts
+    }
+  end
+
+  def set_inboxes
+    @inbox_ids = @current_user.assigned_inboxes.pluck(:id)
+  end
+
+  private
+
+  def filter_conversations
+    @conversations = PgSearch.multisearch((@params[:q]).to_s).where(
+      inbox_id: @inbox_ids, account_id: @current_account, searchable_type: 'Conversation'
+    ).joins('INNER JOIN conversations ON pg_search_documents.searchable_id = conversations.id').includes(:searchable).limit(20).collect(&:searchable)
+  end
+
+  def filter_messages
+    @messages = PgSearch.multisearch((@params[:q]).to_s).where(
+      inbox_id: @inbox_ids, account_id: @current_account, searchable_type: 'Message'
+    ).joins('INNER JOIN messages ON pg_search_documents.searchable_id = messages.id').includes(:searchable).limit(20).collect(&:searchable)
+  end
+
+  def filter_contacts
+    @contacts = PgSearch.multisearch((@params[:q]).to_s).where(
+      account_id: @current_account, searchable_type: 'Contact'
+    ).joins('INNER JOIN contacts ON pg_search_documents.searchable_id = contacts.id').includes(:searchable).limit(20).collect(&:searchable)
+  end
+end

app/javascript/survey/views/Response.vue

@@ -174,7 +174,7 @@ export default {
           feedback_message: this.feedbackMessage,
         };
       } catch (error) {
-        const errorMessage = error?.response?.data?.error;
+        const errorMessage = error?.response?.data?.message;
         this.errorMessage = errorMessage || this.$t('SURVEY.API.ERROR_MESSAGE');
       } finally {
         this.isUpdating = false;

app/jobs/conversations/account_based_search_job.rb

@@ -0,0 +1,9 @@
+class Conversations::AccountBasedSearchJob < ApplicationJob
+  queue_as :async_database_migration
+
+  def perform(account_id)
+    Contact.rebuild_pg_search_documents(account_id)
+    Conversation.rebuild_pg_search_documents(account_id)
+    Message.rebuild_pg_search_documents(account_id)
+  end
+end

app/jobs/conversations/multi_search_job.rb

@@ -0,0 +1,9 @@
+class Conversations::MultiSearchJob < ApplicationJob
+  queue_as :async_database_migration
+
+  def perform
+    Account.all.each do |account|
+      Conversations::AccountBasedSearchJob.perform_later(account.id)
+    end
+  end
+end

app/models/concerns/multi_searchable_helpers.rb

@@ -0,0 +1,74 @@
+module MultiSearchableHelpers
+  extend ActiveSupport::Concern
+
+  included do
+    PgSearch.multisearch_options = {
+      using: {
+        trigram: {
+          word_similarity: true,
+          threshold: 0.3
+        },
+        tsearch: { any_word: true }
+      }
+    }
+
+    def update_contact_search_document
+      return if contact_pg_search_record.present?
+
+      initialize_contact_pg_search_record.update!(
+        content: "#{contact.id} #{contact.email} #{contact.name} #{contact.phone_number} #{contact.account_id}",
+        conversation_id: id,
+        inbox_id: inbox_id
+      )
+    end
+
+    # NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
+    # We can not find conversation_id from contacts directly so we added this joins here.
+    def self.rebuild_pg_search_documents(account_id)
+      return unless self.name == 'Conversation'
+      rebuild_search_documents(account_id)
+    end
+  end
+
+  def rebuild_search_documents(account_id)
+    connection.execute <<~SQL.squish
+      INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
+        SELECT 'Conversation' AS searchable_type,
+                conversations.id AS searchable_id,
+                CONCAT_WS(' ', conversations.display_id, contacts.email, contacts.name, contacts.phone_number, conversations.account_id) AS content,
+                conversations.account_id::int AS account_id,
+                conversations.id::int AS conversation_id,
+                conversations.inbox_id::int AS inbox_id,
+                now() AS created_at,
+                now() AS updated_at
+        FROM conversations
+        INNER JOIN contacts
+          ON conversations.contact_id = contacts.id
+        WHERE conversations.account_id = #{account_id}
+    SQL
+  end
+
+  def contact_pg_search_record
+    contacts_pg_search_records.find_by(conversation_id: id, inbox_id: inbox_id)
+  end
+
+  def initialize_contact_pg_search_record
+    record = contacts_pg_search_records.find_by(conversation_id: nil, inbox_id: nil)
+
+    return record if record.present?
+
+    PgSearch::Document.new(
+      searchable_type: 'Contact',
+      searchable_id: contact_id,
+      account_id: account_id
+    )
+  end
+
+  def contacts_pg_search_records
+    PgSearch::Document.where(
+      searchable_type: 'Contact',
+      searchable_id: contact_id,
+      account_id: account_id
+    )
+  end
+end

app/models/contact.rb

@@ -26,6 +26,13 @@ class Contact < ApplicationRecord
   include Avatarable
   include AvailabilityStatusable
   include Labelable
+  include PgSearch::Model
+  include MultiSearchableHelpers
+
+  multisearchable(
+    against: [:id, :email, :name, :phone_number],
+    additional_attributes: ->(contact) { { conversation_id: nil, account_id: contact.account_id, inbox_id: nil } }
+  )
 
   validates :account_id, presence: true
   validates :email, allow_blank: true, uniqueness: { scope: [:account_id], case_sensitive: false },
@@ -140,6 +147,28 @@ class Contact < ApplicationRecord
     email_format
   end
 
+  # NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
+  # We can not find conversation_id from contacts directly so we added this joins here.
+  def self.rebuild_pg_search_documents(account_id)
+    return super unless name == 'Contact'
+
+    connection.execute <<~SQL.squish
+      INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
+        SELECT 'Contact' AS searchable_type,
+                contacts.id AS searchable_id,
+                CONCAT_WS(' ', contacts.id, contacts.email, contacts.name, contacts.phone_number, contacts.account_id) AS content,
+                contacts.account_id::int AS account_id,
+                conversations.id::int AS conversation_id,
+                conversations.inbox_id::int AS inbox_id,
+                now() AS created_at,
+                now() AS updated_at
+        FROM contacts
+        INNER JOIN conversations
+          ON conversations.contact_id = contacts.id
+        WHERE contacts.account_id = #{account_id}
+    SQL
+  end
+
   private
 
   def ip_lookup

app/models/conversation.rb

@@ -48,7 +48,15 @@ class Conversation < ApplicationRecord
   include ActivityMessageHandler
   include UrlHelper
   include SortHandler
+  include PgSearch::Model
+  include MultiSearchableHelpers
 
+  multisearchable(
+    against: [:display_id, :name, :email, :phone_number, :account_id],
+    additional_attributes: lambda { |conversation|
+                             { conversation_id: conversation.id, account_id: conversation.account_id, inbox_id: conversation.inbox_id }
+                           }
+  )
   validates :account_id, presence: true
   validates :inbox_id, presence: true
   before_validation :validate_additional_attributes
@@ -93,9 +101,11 @@ class Conversation < ApplicationRecord
 
   after_update_commit :execute_after_update_commit_callbacks
   after_create_commit :notify_conversation_creation
+  after_create_commit :update_contact_search_document, if: :contact_id?
   after_commit :set_display_id, unless: :display_id?
 
   delegate :auto_resolve_duration, to: :account
+  delegate :name, :email, :phone_number, to: :contact, allow_nil: true
 
   def can_reply?
     channel = inbox&.channel

app/models/message.rb

@@ -33,6 +33,14 @@
 class Message < ApplicationRecord
   include MessageFilterHelpers
   NUMBER_OF_PERMITTED_ATTACHMENTS = 15
+  include PgSearch::Model
+  include MultiSearchableHelpers
+
+  multisearchable(
+    against: [:content],
+    if: :allowed_message_types?,
+    additional_attributes: ->(message) { { conversation_id: message.conversation_id, account_id: message.account_id, inbox_id: message.inbox_id } }
+  )
 
   before_validation :ensure_content_type
 
@@ -162,6 +170,26 @@ class Message < ApplicationRecord
     true
   end
 
+  # NOTE: To add multi search records with conversation_id associated to contacts for previously added records.
+  # We can not find conversation_id from contacts directly so we added this joins here.
+  def self.rebuild_pg_search_documents(account_id)
+    return super unless name == 'Message'
+
+    connection.execute <<~SQL.squish
+      INSERT INTO pg_search_documents (searchable_type, searchable_id, content, account_id, conversation_id, inbox_id, created_at, updated_at)
+        SELECT 'Message' AS searchable_type,
+                messages.id AS searchable_id,
+                CONCAT_WS(' ', messages.content) AS content,
+                messages.account_id::int AS account_id,
+                messages.conversation_id::int AS conversation_id,
+                messages.inbox_id::int AS inbox_id,
+                now() AS created_at,
+                now() AS updated_at
+        FROM messages
+        WHERE messages.account_id = #{account_id}
+    SQL
+  end
+
   private
 
   def ensure_content_type
@@ -274,4 +302,8 @@ class Message < ApplicationRecord
     conversation.update_columns(last_activity_at: created_at)
     # rubocop:enable Rails/SkipsModelValidations
   end
+
+  def allowed_message_types?
+    incoming? || outgoing?
+  end
 end

app/views/api/v1/accounts/conversations/text_search.json.jbuilder

@@ -0,0 +1,32 @@
+json.payload do
+  json.conversations do
+    json.array! @result[:conversations] do |conversation|
+      json.id conversation.display_id
+      json.account_id conversation.account_id
+      json.created_at conversation.created_at.to_i
+      json.message do
+        json.partial! 'api/v1/models/multi_search_message', formats: [:json], message: conversation.messages.try(:first)
+      end
+      json.contact do
+        json.partial! 'api/v1/models/multi_search_contact', formats: [:json], contact: conversation.contact if conversation.try(:contact).present?
+      end
+      json.inbox do
+        json.partial! 'api/v1/models/multi_search_inbox', formats: [:json], inbox: conversation.inbox if conversation.try(:inbox).present?
+      end
+      json.agent do
+        json.partial! 'api/v1/models/multi_search_agent', formats: [:json], agent: conversation.assignee if conversation.try(:assignee).present?
+      end
+    end
+  end
+  json.contacts do
+    json.array! @result[:contacts] do |contact|
+      json.partial! 'api/v1/models/multi_search_contact', formats: [:json], contact: contact
+    end
+  end
+
+  json.messages do
+    json.array! @result[:messages] do |message|
+      json.partial! 'api/v1/models/multi_search_message', formats: [:json], message: message
+    end
+  end
+end

app/views/api/v1/models/_message.json.jbuilder

@@ -2,7 +2,8 @@ json.id message.id
 json.content message.content
 json.inbox_id message.inbox_id
 json.echo_id message.echo_id if message.echo_id
-json.conversation_id message.conversation.display_id
+# For deleted conversation, messages are not yet deleted [because of destroy_async] for this we added try block
+json.conversation_id message.conversation.try(:display_id)
 json.message_type message.message_type_before_type_cast
 json.content_type message.content_type
 json.status message.status

app/views/api/v1/models/_multi_search_agent.json.jbuilder

@@ -0,0 +1,5 @@
+json.id agent.id
+json.available_name agent.available_name
+json.email agent.email
+json.name agent.name
+json.role agent.role

app/views/api/v1/models/_multi_search_contact.json.jbuilder

@@ -0,0 +1,5 @@
+json.email contact.email
+json.id contact.id
+json.name contact.name
+json.phone_number contact.phone_number
+json.identifier contact.identifier

app/views/api/v1/models/_multi_search_inbox.json.jbuilder

@@ -0,0 +1,4 @@
+json.id inbox.id
+json.channel_id inbox.channel_id
+json.name inbox.name
+json.channel_type inbox.channel_type

app/views/api/v1/models/_multi_search_message.json.jbuilder

@@ -0,0 +1,15 @@
+json.id message.id
+json.content message.content
+json.message_type message.message_type_before_type_cast
+json.content_type message.content_type
+json.source_id message.source_id
+json.inbox_id message.inbox_id
+json.created_at message.created_at.to_i
+json.agent do
+  if message.conversation.try(:assignee).present?
+    json.partial! 'api/v1/models/multi_search_agent', formats: [:json], agent: message.conversation.try(:assignee)
+  end
+end
+json.inbox do
+  json.partial! 'api/v1/models/multi_search_inbox', formats: [:json], inbox: message.inbox if message.inbox.present? && message.try(:inbox).present?
+end

config/routes.rb

@@ -71,6 +71,7 @@ Rails.application.routes.draw do
               get :meta
               get :search
               post :filter
+              get :text_search
             end
             scope module: :conversations do
               resources :messages, only: [:index, :create, :destroy]

config/sidekiq.yml

@@ -12,20 +12,21 @@
 # even put in dynamic logic, like a host-specific queue.
 # http://www.mikeperham.com/2013/11/13/advanced-sidekiq-host-specific-queues/
 :queues:
-  - [low, 1]
+  - [async_database_migration, 1]
-  - [scheduled_jobs, 1]
+  - [low, 2]
-  - [webhooks, 1]
+  - [scheduled_jobs, 2]
-  - [bots, 1]
+  - [webhooks, 2]
-  - [active_storage_analysis, 1]
+  - [bots, 2]
-  - [action_mailbox_incineration, 1]
+  - [active_storage_analysis, 2]
-  - [active_storage_purge, 1]
+  - [action_mailbox_incineration, 2]
-  - [integrations, 2]
+  - [active_storage_purge, 2]
-  - [default, 2]
+  - [integrations, 3]
-  - [mailers, 2]
+  - [default, 3]
-  - [medium, 3]
+  - [mailers, 3]
-  - [events, 3]
+  - [medium, 4]
-  - [action_mailbox_routing, 3]
+  - [events, 4]
-  - [high, 5]
+  - [action_mailbox_routing, 4]
+  - [high, 6]
   - [critical, 10]
 
 # you can override concurrency based on environment

db/migrate/20221205081737_create_pg_search_documents.rb

@@ -0,0 +1,22 @@
+class CreatePgSearchDocuments < ActiveRecord::Migration[6.1]
+  def up
+    say_with_time('Creating table for pg_search multisearch') do
+      create_table :pg_search_documents do |t|
+        t.text :content
+        t.bigint 'conversation_id'
+        t.bigint 'account_id'
+        t.bigint 'inbox_id'
+        t.belongs_to :searchable, polymorphic: true, index: true
+        t.timestamps null: false
+      end
+      add_index :pg_search_documents, :account_id
+      add_index :pg_search_documents, :conversation_id
+    end
+  end
+
+  def down
+    say_with_time('Dropping table for pg_search multisearch') do
+      drop_table :pg_search_documents
+    end
+  end
+end

db/migrate/20221212061802_enable_multi_searchable.rb

@@ -0,0 +1,10 @@
+class EnableMultiSearchable < ActiveRecord::Migration[6.1]
+  def up
+    ::Conversations::MultiSearchJob.perform_now
+    execute 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
+  end
+
+  def down
+    PgSearch::Document.delete_all
+  end
+end

db/schema.rb

@@ -14,6 +14,7 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
+  enable_extension "pg_trgm"
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
 
@@ -399,7 +400,7 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
     t.datetime "agent_last_seen_at"
     t.jsonb "additional_attributes", default: {}
     t.bigint "contact_inbox_id"
-    t.uuid "uuid", default: -> { "gen_random_uuid()" }, null: false
+    t.uuid "uuid", default: -> { "public.gen_random_uuid()" }, null: false
     t.string "identifier"
     t.datetime "last_activity_at", default: -> { "CURRENT_TIMESTAMP" }, null: false
     t.bigint "team_id"
@@ -674,6 +675,20 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
     t.index ["user_id"], name: "index_notifications_on_user_id"
   end
 
+  create_table "pg_search_documents", force: :cascade do |t|
+    t.text "content"
+    t.bigint "conversation_id"
+    t.bigint "account_id"
+    t.bigint "inbox_id"
+    t.string "searchable_type"
+    t.bigint "searchable_id"
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["account_id"], name: "index_pg_search_documents_on_account_id"
+    t.index ["conversation_id"], name: "index_pg_search_documents_on_conversation_id"
+    t.index ["searchable_type", "searchable_id"], name: "index_pg_search_documents_on_searchable"
+  end
+
   create_table "platform_app_permissibles", force: :cascade do |t|
     t.bigint "platform_app_id", null: false
     t.string "permissible_type", null: false
@@ -836,6 +851,9 @@ ActiveRecord::Schema.define(version: 2022_12_19_162759) do
     t.jsonb "custom_attributes", default: {}
     t.string "type"
     t.text "message_signature"
+    t.datetime "locked_at"
+    t.integer "failed_attempts"
+    t.string "unlock_token"
     t.index ["email"], name: "index_users_on_email"
     t.index ["pubsub_token"], name: "index_users_on_pubsub_token", unique: true
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true

spec/finders/text_search_spec.rb

@@ -0,0 +1,87 @@
+require 'rails_helper'
+
+describe ::TextSearch do
+  subject(:text_search) { described_class.new(user_1, params) }
+
+  let!(:account) { create(:account) }
+  let!(:user_1) { create(:user, account: account) }
+  let!(:user_2) { create(:user, account: account) }
+  let!(:inbox) { create(:inbox, account: account, enable_auto_assignment: false) }
+
+  before do
+    create(:inbox_member, user: user_1, inbox: inbox)
+    create(:inbox_member, user: user_2, inbox: inbox)
+
+    create(:contact, name: '1223', account_id: account.id)
+    create(:contact, name: 'Potter', account_id: account.id)
+    contact_2 = create(:contact, name: 'Harry Potter', account_id: account.id, email: 'harry@chatwoot.com')
+    conversation_1 = create(:conversation, account: account, inbox: inbox, assignee: user_1, display_id: 1213)
+    conversation_2 = create(:conversation, account: account, inbox: inbox, assignee: user_1, display_id: 1223)
+    create(:conversation, account: account, inbox: inbox, assignee: user_1, status: 'resolved', display_id: 13, contact_id: contact_2.id)
+    create(:conversation, account: account, inbox: inbox, assignee: user_2, display_id: 14)
+    create(:conversation, account: account, inbox: inbox, display_id: 15)
+    Current.account = account
+
+    create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'Ask Lisa')
+    create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'message_12')
+    create(:message, conversation_id: conversation_1.id, account_id: account.id, content: 'message_13')
+
+    create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'Pottery Barn order')
+    create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'message_22')
+    create(:message, conversation_id: conversation_2.id, account_id: account.id, content: 'message_23')
+  end
+
+  describe '#perform' do
+    context 'with text search' do
+      it 'filter conversations by number' do
+        params = { q: '122' }
+        result = described_class.new(user_1, params).perform
+        expect(result[:conversations].length).to eq 1
+        expect(result[:contacts].length).to eq 1
+      end
+
+      it 'filter message and contacts by string' do
+        params = { q: 'pot' }
+        result = described_class.new(user_1, params).perform
+        expect(result[:messages].length).to be 1
+        expect(result[:contacts].length).to be 2
+      end
+
+      it 'filter conversations by contact details' do
+        params = { q: 'pot' }
+        result = described_class.new(user_1, params).perform
+        expect(result[:conversations].length).to be 1
+      end
+
+      it 'filter conversations by contact email' do
+        params = { q: 'harry@chatwoot.com' }
+        result = described_class.new(user_1, params).perform
+        expect(result[:conversations].length).to be 1
+      end
+    end
+
+    context 'when create records in tables including multi search' do
+      let(:contact) { create(:contact, name: 'Welma', account_id: account.id, email: 'welma@scoobydoo.com') }
+      let(:conversation) { create(:conversation, account: account, inbox: inbox, assignee: user_1, status: 'open', contact_id: contact.id) }
+
+      it 'conversation creation pg search records' do
+        contact_search_record = PgSearch::Document.find_by(searchable_id: contact.id, searchable_type: contact.class.name)
+        conversation_search_record = PgSearch::Document.find_by(searchable_id: conversation.id, searchable_type: conversation.class.name)
+
+        expect(contact_search_record).to be_present
+        expect(conversation_search_record).to be_present
+      end
+
+      it 'conversation deletion deletes pg search records' do
+        contact.destroy!
+        conversation.destroy!
+
+        contact_search_record = PgSearch::Document.find_by(searchable_id: contact.id, searchable_type: contact.class.name)
+        conversation_search_record = PgSearch::Document.find_by(searchable_id: conversation.id, searchable_type: conversation.class.name)
+
+        expect(contact_search_record).to be_nil
+        expect(conversation_search_record).to be_nil
+      end
+    end
+  end
+end