feat: IP lookup (#1315)

- feature to store contact IP for accounts
- IP lookup through geocoder gem
- ability to do IP lookup through external APIs
- add commit hook to prevent push to develop and master
- migrations to fix default values for jsonb columns

.env.example

@@ -117,6 +117,17 @@ IOS_APP_ID=6C953F3RX2.com.chatwoot.app
 ## Bot Customizations
 USE_INBOX_AVATAR_FOR_BOT=true
 
+
+
+## IP look up configuration
+## ref https://github.com/alexreisner/geocoder/blob/master/README_API_GUIDE.md
+## works only on accounts with ip look up feature enabled
+# IP_LOOKUP_SERVICE=geoip2
+# maxmindb api key to use geoip2 service
+# IP_LOOKUP_API_KEY=
+
 ## Development Only Config
 # if you want to use letter_opener for local emails
 # LETTER_OPENER=true
+
+

.gitignore

@@ -16,6 +16,7 @@
 /tmp/*
 !/log/.keep
 !/tmp/.keep
+*.mmdb
 
 # Ignore Byebug command history file.
 .byebug_history

Gemfile

@@ -90,6 +90,12 @@ gem 'sidekiq'
 gem 'fcm'
 gem 'webpush'
 
+##-- geocoding / parse location from ip --##
+# http://www.rubygeocoder.com/
+gem 'geocoder'
+# to parse maxmind db
+gem 'maxminddb'
+
 group :development do
   gem 'annotate'
   gem 'bullet'

Gemfile.lock

@@ -201,6 +201,7 @@ GEM
     ffi (1.13.1)
     flag_shih_tzu (0.3.23)
     foreman (0.87.2)
+    geocoder (1.6.3)
     gli (2.19.2)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -290,6 +291,7 @@ GEM
       mini_mime (>= 0.1.1)
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
+    maxminddb (0.1.22)
     memoist (0.16.2)
     method_source (1.0.0)
     mime-types (3.3.1)
@@ -578,6 +580,7 @@ DEPENDENCIES
   fcm
   flag_shih_tzu
   foreman
+  geocoder
   google-cloud-storage
   groupdate
   haikunator
@@ -590,6 +593,7 @@ DEPENDENCIES
   letter_opener
   liquid
   listen
+  maxminddb
   mini_magick
   mock_redis!
   pg

app/controllers/api/v1/accounts/contacts_controller.rb

@@ -19,13 +19,16 @@ class Api::V1::Accounts::ContactsController < Api::V1::Accounts::BaseController
   def create
     ActiveRecord::Base.transaction do
       @contact = Current.account.contacts.new(contact_params)
+      set_ip
       @contact.save!
       @contact_inbox = build_contact_inbox
     end
   end
 
   def update
-    @contact.update!(contact_update_params)
+    @contact.assign_attributes(contact_update_params)
+    set_ip
+    @contact.save!
   rescue ActiveRecord::RecordInvalid => e
     render json: {
       message: e.record.errors.full_messages.join(', '),
@@ -67,4 +70,11 @@ class Api::V1::Accounts::ContactsController < Api::V1::Accounts::BaseController
   def fetch_contact
     @contact = Current.account.contacts.includes(contact_inboxes: [:inbox]).find(params[:id])
   end
+
+  def set_ip
+    return if @contact.account.feature_enabled?('ip_lookup')
+
+    @contact[:additional_attributes][:created_at_ip] ||= request.remote_ip
+    @contact[:additional_attributes][:updated_at_ip] = request.remote_ip
+  end
 end

app/jobs/contact_ip_lookup_job.rb

@@ -0,0 +1,57 @@
+require 'rubygems/package'
+
+class ContactIpLookupJob < ApplicationJob
+  queue_as :default
+
+  def perform(contact)
+    return unless ensure_look_up_service
+
+    update_contact_location_from_ip(contact)
+  rescue Errno::ETIMEDOUT => e
+    Rails.logger.info "Exception: ip resolution failed : #{e.message}"
+  end
+
+  private
+
+  def ensure_look_up_service
+    return if ENV['IP_LOOKUP_SERVICE'].blank? || ENV['IP_LOOKUP_API_KEY'].blank?
+    return true if ENV['IP_LOOKUP_SERVICE'].to_sym != :geoip2
+
+    ensure_look_up_db
+  end
+
+  def update_contact_location_from_ip(contact)
+    ip = get_contact_ip(contact)
+    return if ip.blank?
+
+    contact.additional_attributes['city'] = Geocoder.search(ip).first.city
+    contact.additional_attributes['country'] = Geocoder.search(ip).first.country
+    contact.save!
+  end
+
+  def get_contact_ip(contact)
+    contact.additional_attributes['updated_at_ip'] || contact.additional_attributes['created_at_ip']
+  end
+
+  def ensure_look_up_db
+    return true if File.exist?(GeocoderConfiguration::LOOK_UP_DB)
+
+    setup_vendor_db
+  end
+
+  def setup_vendor_db
+    base_url = 'https://download.maxmind.com/app/geoip_download'
+    source = URI.open("#{base_url}?edition_id=GeoLite2-City&suffix=tar.gz&license_key=#{ENV['IP_LOOKUP_API_KEY']}")
+    tar_extract = Gem::Package::TarReader.new(Zlib::GzipReader.open(source))
+    tar_extract.rewind
+
+    tar_extract.each do |entry|
+      next unless entry.full_name.include?('GeoLite2-City.mmdb') && entry.file?
+
+      File.open GeocoderConfiguration::LOOK_UP_DB, 'wb' do |f|
+        f.print entry.read
+      end
+      return true
+    end
+  end
+end

app/models/contact.rb

@@ -40,6 +40,7 @@ class Contact < ApplicationRecord
   before_validation :prepare_email_attribute
   after_create_commit :dispatch_create_event
   after_update_commit :dispatch_update_event
+  after_commit :ip_lookup
 
   def get_source_id(inbox_id)
     contact_inboxes.find_by!(inbox_id: inbox_id).source_id
@@ -68,6 +69,12 @@ class Contact < ApplicationRecord
     }
   end
 
+  def ip_lookup
+    return unless account.feature_enabled?('ip_lookup')
+
+    ContactIpLookupJob.perform_later(self)
+  end
+
   def prepare_email_attribute
     # So that the db unique constraint won't throw error when email is ''
     self.email = nil if email.blank?

bin/validate_push

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# script prevents pushing to develop and master by mistake
+
+branch="$(git rev-parse --abbrev-ref HEAD)"
+
+if [ "$branch" = "master" ]; then
+  echo "You can't push directly to master branch"
+  exit 1
+elif [ "$branch" = "develop" ]; then
+  echo "You can't push directly to develop branch"
+  exit 1
+fi

config/features.yml

@@ -7,3 +7,5 @@
   enabled: true
 - name: channel_twitter
   enabled: true
+- name: ip_lookup
+  enabled: false

config/initializers/geocoder.rb

@@ -0,0 +1,32 @@
+# Geocoding options
+# timeout: 3,                 # geocoding service timeout (secs)
+# lookup: :nominatim,         # name of geocoding service (symbol)
+# ip_lookup: :ipinfo_io,      # name of IP address geocoding service (symbol)
+# language: :en,              # ISO-639 language code
+# use_https: false,           # use HTTPS for lookup requests? (if supported)
+# http_proxy: nil,            # HTTP proxy server (user:pass@host:port)
+# https_proxy: nil,           # HTTPS proxy server (user:pass@host:port)
+# api_key: nil,               # API key for geocoding service
+# cache: nil,                 # cache object (must respond to #[], #[]=, and #del)
+# cache_prefix: 'geocoder:',  # prefix (string) to use for all cache keys
+
+# Exceptions that should not be rescued by default
+# (if you want to implement custom error handling);
+# supports SocketError and Timeout::Error
+# always_raise: [],
+
+# Calculation options
+# units: :mi,                 # :km for kilometers or :mi for miles
+# distances: :linear          # :spherical or :linear
+
+module GeocoderConfiguration
+  LOOK_UP_DB = Rails.root.join('vendor/db/GeoLiteCity.mmdb')
+end
+
+if ENV['IP_LOOKUP_SERVICE'].present?
+  if ENV['IP_LOOKUP_SERVICE'] == 'geoip2'
+    Geocoder.configure(ip_lookup: :geoip2, geoip2: { file: GeocoderConfiguration::LOOK_UP_DB })
+  else
+    Geocoder.configure(ip_lookup: ENV['IP_LOOKUP_SERVICE'].to_sym, api_key: ENV['IP_LOOKUP_API_KEY'])
+  end
+end

db/migrate/20201019173944_add_default_value_to_jsonb_colums.rb

@@ -0,0 +1,8 @@
+class AddDefaultValueToJsonbColums < ActiveRecord::Migration[6.0]
+  def change
+    change_column_default :contacts, :additional_attributes, from: nil, to: {}
+    change_column_default :conversations, :additional_attributes, from: nil, to: {}
+    change_column_default :installation_configs, :serialized_value, from: '{}', to: {}
+    change_column_default :notification_subscriptions, :subscription_attributes, from: '{}', to: {}
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_10_11_152227) do
+ActiveRecord::Schema.define(version: 2020_10_19_173944) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "pg_stat_statements"
@@ -203,7 +203,7 @@ ActiveRecord::Schema.define(version: 2020_10_11_152227) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.string "pubsub_token"
-    t.jsonb "additional_attributes"
+    t.jsonb "additional_attributes", default: {}
     t.string "identifier"
     t.jsonb "custom_attributes", default: {}
     t.index ["account_id"], name: "index_contacts_on_account_id"
@@ -224,7 +224,7 @@ ActiveRecord::Schema.define(version: 2020_10_11_152227) do
     t.datetime "contact_last_seen_at"
     t.datetime "agent_last_seen_at"
     t.boolean "locked", default: false
-    t.jsonb "additional_attributes"
+    t.jsonb "additional_attributes", default: {}
     t.bigint "contact_inbox_id"
     t.uuid "uuid", default: -> { "gen_random_uuid()" }, null: false
     t.string "identifier"
@@ -285,7 +285,7 @@ ActiveRecord::Schema.define(version: 2020_10_11_152227) do
 
   create_table "installation_configs", force: :cascade do |t|
     t.string "name", null: false
-    t.jsonb "serialized_value", default: "{}", null: false
+    t.jsonb "serialized_value", default: {}, null: false
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.index ["name", "created_at"], name: "index_installation_configs_on_name_and_created_at", unique: true
@@ -399,7 +399,7 @@ ActiveRecord::Schema.define(version: 2020_10_11_152227) do
   create_table "notification_subscriptions", force: :cascade do |t|
     t.bigint "user_id", null: false
     t.integer "subscription_type", null: false
-    t.jsonb "subscription_attributes", default: "{}", null: false
+    t.jsonb "subscription_attributes", default: {}, null: false
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
     t.string "identifier"
@@ -452,9 +452,11 @@ ActiveRecord::Schema.define(version: 2020_10_11_152227) do
     t.index ["taggable_id", "taggable_type", "context"], name: "index_taggings_on_taggable_id_and_taggable_type_and_context"
     t.index ["taggable_id", "taggable_type", "tagger_id", "context"], name: "taggings_idy"
     t.index ["taggable_id"], name: "index_taggings_on_taggable_id"
+    t.index ["taggable_type", "taggable_id"], name: "index_taggings_on_taggable_type_and_taggable_id"
     t.index ["taggable_type"], name: "index_taggings_on_taggable_type"
     t.index ["tagger_id", "tagger_type"], name: "index_taggings_on_tagger_id_and_tagger_type"
     t.index ["tagger_id"], name: "index_taggings_on_tagger_id"
+    t.index ["tagger_type", "tagger_id"], name: "index_taggings_on_tagger_type_and_tagger_id"
   end
 
   create_table "tags", id: :serial, force: :cascade do |t|

package.json

@@ -90,7 +90,8 @@
   },
   "husky": {
     "hooks": {
-      "pre-commit": "lint-staged"
+      "pre-commit": "lint-staged",
+      "pre-push": "sh bin/validate_push"
     }
   },
   "jest": {

spec/models/conversation_spec.rb

@@ -377,7 +377,7 @@ RSpec.describe Conversation, type: :model do
     let(:conversation) { create(:conversation) }
     let(:expected_data) do
       {
-        additional_attributes: nil,
+        additional_attributes: {},
         meta: {
           sender: conversation.contact.push_event_data,
           assignee: conversation.assignee

spec/presenters/conversations/event_data_presenter_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Conversations::EventDataPresenter do
   describe '#push_data' do
     let(:expected_data) do
       {
-        additional_attributes: nil,
+        additional_attributes: {},
         meta: {
           sender: conversation.contact.push_event_data,
           assignee: conversation.assignee