Class: Ronin::DB::URL

Inherits:
ActiveRecord::Base
  • Object
show all
Includes:
Model, Model::Importable, Model::LastScannedAt
Defined in:
lib/ronin/db/url.rb

Overview

Represents parsed URLs.

Constant Summary collapse

SCHEMES =

Mapping of URL Schemes and URI classes

{
  'https' => ::URI::HTTPS,
  'http'  => ::URI::HTTP,
  'ftp'   => ::URI::FTP
}

Instance Attribute Summary collapse

Attributes included from Model::LastScannedAt

#last_scanned_at

Class Method Summary collapse

Instance Method Summary collapse

Methods included from Model::LastScannedAt

included

Methods included from Model::Importable

included

Methods included from Model

included

Instance Attribute Details

#created_at ⇒ Time (readonly)

Defines the created_at timestamp

Returns:

  • (Time) 


98
 
# File 'lib/ronin/db/url.rb', line 98

attribute :created_at, :datetime

#credentials ⇒ Array<Credentials>

The credentials that will work with this URL.

Returns:

  • (Array<Credentials>) 


117
 
# File 'lib/ronin/db/url.rb', line 117

has_many :credentials, through: :web_credentials

#fragment ⇒ String?

The fragment of the URL.

Returns:

  • (String, nil) 


92
 
# File 'lib/ronin/db/url.rb', line 92

attribute :fragment, :string

#host_name ⇒ HostName

The host name of the URL

Returns:



67
 
# File 'lib/ronin/db/url.rb', line 67

belongs_to :host_name, required: true

#id ⇒ Integer

The primary key of the URL.

Returns:

  • (Integer) 


54
 
# File 'lib/ronin/db/url.rb', line 54

attribute :id, :integer

#path ⇒ String

The path of the URL.

Returns:

  • (String) 


80
 
# File 'lib/ronin/db/url.rb', line 80

attribute :path, :string

#port ⇒ Port?

The port of the URL.

Returns:



73
74
 
# File 'lib/ronin/db/url.rb', line 73

belongs_to :port, optional:   true,
class_name: 'Port'

#query ⇒ String?

The query string part of the URL.

Returns:

  • (String, nil) 


86
 
# File 'lib/ronin/db/url.rb', line 86

attribute :query, :string

#query_params ⇒ Array<URLQueryParam>

The query params of the URL.

Returns:



104
105
 
# File 'lib/ronin/db/url.rb', line 104

has_many :query_params, class_name: 'URLQueryParam',
dependent:  :destroy

#scheme ⇒ URLScheme

The scheme of the URL.

Returns:



60
61
 
# File 'lib/ronin/db/url.rb', line 60

belongs_to :scheme, required:   true,
class_name: 'URLScheme'

#web_credentials ⇒ Array<WebCredential>

Any credentials used with the URL.

Returns:



111
 
# File 'lib/ronin/db/url.rb', line 111

has_many :web_credentials, dependent: :destroy

Class Method Details

.http ⇒ Array<URL>

Searches for all URLs using HTTP.

Returns:

  • (Array<URL>) —

    The matching URLs.



127
128
129
 
# File 'lib/ronin/db/url.rb', line 127

def self.http
  joins(:scheme).where(scheme: {name: 'http'})
end

.https ⇒ Array<URL>

Searches for all URLs using HTTPS.

Returns:

  • (Array<URL>) —

    The matching URLs.



139
140
141
 
# File 'lib/ronin/db/url.rb', line 139

def self.https
  joins(:scheme).where(scheme: {name: 'https'})
end

.import(uri) ⇒ URL

Creates a new URL.

Parameters:

  • uri (String, URI::HTTP) —

    The URI to create the URL from.

Returns:

  • (URL) —

    The new URL.



353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
 
# File 'lib/ronin/db/url.rb', line 353

def self.import(uri)
  uri = URI(uri)

  # find or create the URL scheme, host_name and port
  scheme    = URLScheme.find_or_create_by(name: uri.scheme)
  host_name = HostName.find_or_create_by(name: uri.host)
  port      = if uri.port
                Port.find_or_create_by(
                  protocol: :tcp,
                  number:   uri.port
                )
              end
  path      = normalized_path(uri)
  query     = uri.query
  fragment  = uri.fragment

  # try to query a pre-existing URI then fallback to creating the URL
  # with query params.
  return find_or_create_by(
    scheme:       scheme,
    host_name:    host_name,
    port:         port,
    path:         path,
    query:        query,
    fragment:     fragment
  ) do |new_url|
    if uri.respond_to?(:query_params)
      # find or create the URL query params
      uri.query_params.each do |name,value|
        new_url.query_params << URLQueryParam.new(
          name:  URLQueryParamName.find_or_create_by(name: name),
          value: value
        )
      end
    end
  end
end

.lookup(url) ⇒ URL?

Searches for a URL.

Parameters:

  • url (URI::HTTP, String) —

    The URL to search for.

Returns:

  • (URL, nil) —

    The matching URL.



322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
 
# File 'lib/ronin/db/url.rb', line 322

def self.lookup(url)
  uri = URI(url)

  # create the initial query
  query = joins(:scheme, :host_name).where(
    scheme:    {name: uri.scheme},
    host_name: {name: uri.host},
    path:      normalized_path(uri),
    query:     uri.query,
    fragment:  uri.fragment
  )

  if uri.port
    # query the port
    query = query.joins(:port).where(port: {number: uri.port})
  end

  return query.first
end

.normalized_path(uri) ⇒ String?

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Normalizes the path of a URI.

Parameters:

  • uri (URI) —

    The URI containing the path.

Returns:

  • (String, nil) —

    The normalized path.



473
474
475
476
477
478
479
480
481
482
483
 
# File 'lib/ronin/db/url.rb', line 473

def self.normalized_path(uri)
  case uri
  when ::URI::HTTP
    # map empty HTTP paths to '/'
    unless uri.path.empty? then uri.path
    else                        '/'
    end
  else
    uri.path
  end
end

.with_basename(basename) ⇒ Array<URL>

Searches for all URLs sharing a common base name.

Parameters:

  • basename (String) —

    The base name to search for.

Returns:

  • (Array<URL>) —

    The URL with the common base name.



231
232
233
234
235
 
# File 'lib/ronin/db/url.rb', line 231

def self.with_basename(basename)
  path_column = self.arel_table[:path]

  where(path_column.matches("%/#{basename}"))
end

.with_directory(root_dir) ⇒ Array<URL>

Searches for all URLs sharing a common sub-directory.

Parameters:

  • root_dir (String) —

    The sub-directory to search for.

Returns:

  • (Array<URL>) —

    The URL with the common sub-directory.



214
215
216
217
218
 
# File 'lib/ronin/db/url.rb', line 214

def self.with_directory(root_dir)
  path_column = self.arel_table[:path]

  where(path: root_dir).or(where(path_column.matches("#{root_dir}/%")))
end

.with_file_ext(ext) ⇒ Array<URL>

Searches for all URLs with a common file-extension.

Parameters:

  • ext (String) —

    The file extension to search for.

Returns:

  • (Array<URL>) —

    The URLs with the common file-extension.



248
249
250
251
252
 
# File 'lib/ronin/db/url.rb', line 248

def self.with_file_ext(ext)
  path_column = self.arel_table[:path]

  where(path_column.matches("%.#{sanitize_sql_like(ext)}"))
end

.with_fragment(fragment) ⇒ Array<URL>

Searches for all URLs with the exact fragment.

Parameters:

  • fragment (String) —

    The fragment to search for.

Returns:

  • (Array<URL>) —

    The URL with the matching fragment.



199
200
201
 
# File 'lib/ronin/db/url.rb', line 199

def self.with_fragment(fragment)
  where(fragment: fragment)
end

.with_host_name(name) ⇒ Array<URL>

Searches for URLs with specific host name(s).

Parameters:

  • name (String, Array<String>) —

    The host name(s) to search for.

Returns:

  • (Array<URL>) —

    The matching URLs.



154
155
156
 
# File 'lib/ronin/db/url.rb', line 154

def self.with_host_name(name)
  joins(:host_name).where(host_name: {name: name})
end

.with_path(path) ⇒ Array<URL>

Searches for all URLs with the exact path.

Parameters:

  • path (String) —

    The path to search for.

Returns:

  • (Array<URL>) —

    The URL with the matching path.



184
185
186
 
# File 'lib/ronin/db/url.rb', line 184

def self.with_path(path)
  where(path: path)
end

.with_port_number(number) ⇒ Array<URL>

Searches for URLs with the specific port number(s).

Parameters:

  • number (Integer, Array<Integer>) —

    The port number(s) to search for.

Returns:

  • (Array<URL>) —

    The matching URLs.



169
170
171
 
# File 'lib/ronin/db/url.rb', line 169

def self.with_port_number(number)
  joins(:port).where(port: {number: number})
end

.with_query_param(name, value) ⇒ Array<URL>

Searches for URLs with the given query param name and value.

Parameters:

  • name (String, Array<String>) —

    The query param name to search for.

  • value (String, Array<String>) —

    The query param value to search for.

Returns:

  • (Array<URL>) —

    The URLs with the given query param.



268
269
270
271
272
273
274
275
 
# File 'lib/ronin/db/url.rb', line 268

def self.with_query_param(name,value)
  joins(query_params: :name).where(
    query_params: {
      ronin_url_query_param_names:  {name: name},
      value: value
    }
  )
end

.with_query_param_name(name) ⇒ Array<URL>

Search for all URLs with a given query param name.

Parameters:

  • name (Array<String>, String) —

    The query param name to search for.

Returns:

  • (Array<URL>) —

    The URLs with the given query param name.



288
289
290
291
292
293
294
 
# File 'lib/ronin/db/url.rb', line 288

def self.with_query_param_name(name)
  joins(query_params: [:name]).where(
    query_params: {
      ronin_url_query_param_names: {name: name}
    }
  )
end

.with_query_param_value(value) ⇒ Array<URL>

Search for all URLs with a given query param value.

Parameters:

  • value (Array<String>, String) —

    The query param value to search for.

Returns:

  • (Array<URL>) —

    The URLs with the given query param value.



307
308
309
 
# File 'lib/ronin/db/url.rb', line 307

def self.with_query_param_value(value)
  joins(:query_params).where(query_params: {value: value})
end

Instance Method Details

#host ⇒ String

The host name of the URL.

Returns:

  • (String) —

    The address of host name.



399
400
401
 
# File 'lib/ronin/db/url.rb', line 399

def host
  self.host_name.name
end

#port_number ⇒ Integer?

The port number used by the URL.

Returns:

  • (Integer, nil) —

    The port number.



411
412
413
 
# File 'lib/ronin/db/url.rb', line 411

def port_number
  self.port.number if self.port
end

#to_s ⇒ String

Converts the URL to a String.

Returns:

  • (String) —

    The string form of the URL.



458
459
460
 
# File 'lib/ronin/db/url.rb', line 458

def to_s
  self.to_uri.to_s
end

#to_uri ⇒ URI::HTTP, URI::HTTPS

Builds a URI object from the URL.

Returns:

  • (URI::HTTP, URI::HTTPS) —

    The URI object created from the URL attributes.



423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
 
# File 'lib/ronin/db/url.rb', line 423

def to_uri
  # map the URL scheme to a URI class
  url_class = SCHEMES.fetch(self.scheme.name,::URI::Generic)

  scheme = if self.scheme
             self.scheme.name
           end

  host = if self.host_name
           self.host_name.name
         end

  port = if self.port
           self.port.number
         end

  # build the URI
  return url_class.build(
    scheme:   scheme,
    host:     host,
    port:     port,
    path:     self.path,
    query:    self.query,
    fragment: self.fragment
  )
end