Brain Book: April 2013

SARG INSTALLATION AND CONFIGURATION ON UBUNTU 12

NOTE: web server is required to view reports.

apt-get install apache2

apt-get install sarg

nano /etc/sarg/sarg.conf

root@raabtaproxy:/etc/sarg# cat sarg.conf

# sarg.conf

# TAG: access_log file

# Where is the access.log file

# sarg -l file

access_log /var/log/squid3/access.log

# TAG: graphs yes|no

# Use graphics where is possible.

graphs yes

graph_days_bytes_bar_color orange

# TAG: graph_font

# The full path to the TTF font file to use to create the graphs. It is required

# if graphs is set to yes.

#graph_font /usr/share/fonts/truetype/ttf-dejavu/DejaVuSans.ttf

# TAG: title

# Especify the title for html page.

title "Raabta Proxy User Access Reports"

# TAG: font_face

# Especify the font for html page.

font_face Tahoma,Verdana,Arial

# TAG: header_color

# Especify the header color

header_color darkblue

# TAG: header_bgcolor

# Especify the header bgcolor

header_bgcolor blanchedalmond

# TAG: font_size

# Especify the text font size

font_size 9px

# TAG: header_font_size

# Especify the header font size

#header_font_size 9px

# TAG: title_font_size

# Especify the title font size

#title_font_size 11px

# TAG: background_color

# Html page background color

background_color white

# TAG: text_color

# Html page text color

text_color #000000

# TAG: text_bgcolor

# Html page text background color

text_bgcolor lavender

# TAG: title_color

# Html page title color

title_color green

# TAG: logo_image

# Html page logo.

#logo_image none

# TAG: logo_text

# Html page logo text.

#logo_text ""

# TAG: logo_text_color

# Html page logo texti color.

#logo_text_color #000000

# TAG: logo_image_size

# Html page logo image size.

# width height

#image_size 80 45

# TAG: background_image

# Html page background image

#background_image none

# TAG: password

# User password file used by Squid authentication scheme

# If used, generate reports just for that users.

#password none

# TAG: temporary_dir

# Temporary directory name for work files

# sarg -w dir

temporary_dir /tmp

# TAG: output_dir

# The reports will be saved in that directory

# sarg -o dir

#output_dir /var/www/html/squid-reports

output_dir /var/www/squid-reports

# TAG: output_email

# Email address to send the reports. If you use this tag, no html reports will be generated.

# sarg -e email

#output_email none

# TAG: resolve_ip yes/no

# Convert ip address to dns name

# sarg -n

resolve_ip yes

# TAG: user_ip yes/no

# Use Ip Address instead userid in reports.

# sarg -p

user_ip no

# TAG: topuser_sort_field field normal/reverse

# Sort field for the Topuser Report.

# Allowed fields: USER CONNECT BYTES TIME

topuser_sort_field BYTES reverse

# TAG: user_sort_field field normal/reverse

# Sort field for the User Report.

# Allowed fields: SITE CONNECT BYTES TIME

user_sort_field BYTES reverse

# TAG: exclude_users file

# users within the file will be excluded from reports.

# you can use indexonly to have only index.html file.

exclude_users /etc/sarg/exclude_users

# TAG: exclude_hosts file

# Hosts, domains or subnets will be excluded from reports.

# Eg.: 192.168.10.10 - exclude ip address only

# 192.168.10.0/24 - exclude full C class

# s1.acme.foo - exclude hostname only

# *.acme.foo - exclude full domain name

exclude_hosts /etc/sarg/exclude_hosts

# TAG: useragent_log file

# useragent.log file patch to generate useragent report.

#useragent_log none

# TAG: date_format

# Date format in reports: e (European=dd/mm/yy), u (American=mm/dd/yy), w (Weekly=yy.ww)

date_format u

# TAG: per_user_limit file MB

# Saves userid on file if download exceed n MB.

# This option allow you to disable user access if user exceed a download limit.

#per_user_limit none

# TAG: lastlog n

# How many reports files must be keept in reports directory.

# The oldest report file will be automatically removed.

# 0 - no limit.

lastlog 0

# TAG: remove_temp_files yes

# Remove temporary files: geral, usuarios, top, periodo from root report directory.

remove_temp_files yes

# TAG: index yes|no|only

# Generate the main index.html.

# only - generate only the main index.html

index yes

# TAG: index_tree date|file

# How to generate the index.

index_tree file

# TAG: overwrite_report yes|no

# yes - if report date already exist then will be overwrited.

# no - if report date already exist then will be renamed to filename.n, filename.n+1

overwrite_report no

# TAG: records_without_userid ignore|ip|everybody

# What can I do with records without user id (no authentication) in access.log file ?

# ignore - This record will be ignored.

# ip - Use ip address instead. (default)

# everybody - Use "everybody" instead.

records_without_userid ip

# TAG: use_comma no|yes

# Use comma instead point in reports.

# Eg.: use_comma yes => 23,450,110

# use_comma no => 23.450.110

use_comma yes

# TAG: mail_utility mail|mailx

# Mail command to use to send reports via SMTP

mail_utility mailx

# TAG: topsites_num n

# How many sites in topsites report.

topsites_num 100

# TAG: topsites_sort_order CONNECT|BYTES A|D

# Sort for topsites report, where A=Ascendent, D=Descendent

topsites_sort_order CONNECT D

# TAG: index_sort_order A/D

# Sort for index.html, where A=Ascendent, D=Descendent

index_sort_order D

# TAG: exclude_codes file

# Ignore records with these codes. Eg.: NONE/400

# Write one code per line. Lines starting with a # are ignored.

# Only codes matching exactly one of the line is rejected. The

# comparison is not case sensitive.

exclude_codes /etc/sarg/exclude_codes

# TAG: replace_index string

# Replace "index.html" in the main index file with this string

# If null "index.html" is used

#replace_index <?php echo str_replace(".", "_", $REMOTE_ADDR); echo ".html"; ?>

# TAG: max_elapsed milliseconds

# If elapsed time is recorded in log is greater than max_elapsed use 0 for elapsed time.

# Use 0 for no checking

max_elapsed 28800000

# 8 Hours

# TAG: report_type type

# What kind of reports to generate.

# topusers - users, sites, times, bytes, connects, links to accessed sites, etc

# topsites - site, connect and bytes report

# sites_users - users and sites report

# users_sites - accessed sites by the user report

# date_time - bytes used per day and hour report

# denied - denied sites with full URL report

# auth_failures - autentication failures report

# site_user_time_date - sites, dates, times and bytes report

# downloads - downloads per user report

# Eg.: report_type topsites denied

#report_type topusers topsites sites_users users_sites date_time denied auth_failures site_user_time_date downloads

report_type topusers topsites sites_users users_sites date_time denied auth_failures site_user_time_date downloads

# TAG: usertab filename

# You can change the "userid" or the "ip address" to be a real user name on the reports.

# If resolve_ip is active, the ip address is resolved before being looked up into this

# file. That is, if you want to map the ip address, be sure to set resolv_ip to no or

# the resolved name will be looked into the file instead of the ip address. Note that

# it can be used to resolve any ip address known to the dns and then map the unresolved

# ip addresses to a name found in the usertab file.

# Table syntax:

# userid name or ip address name

# Eg:

# SirIsaac Isaac Newton

# vinci Leonardo da Vinci

# 192.168.10.1 Karol Wojtyla

# Each line must be terminated with '\n'

# If usertab have value "ldap" (case ignoring), user names

# will be taken from LDAP server. This method as approaches for reception

# of usernames from Active Didectory

usertab /etc/sarg/usertab

# TAG: LDAPHost hostname

# FQDN or IP address of host with LDAP service or AD DC

# default is '127.0.0.1'

#LDAPHost 127.0.0.1

# TAG: LDAPPort port

# LDAP service port number

# default is '389'

#LDAPPort 389

# TAG: LDAPBindDN CN=username,OU=group,DC=mydomain,DC=com

# DN of LDAP user, who is authorized to read user's names from LDAP base

# default is empty line

#LDAPBindDN cn=proxy,dc=mydomain,dc=local

# TAG: LDAPBindPW secret

# Password of DN, who is authorized to read user's names from LDAP base

# default is empty line

#LDAPBindPW secret

# TAG: LDAPBaseSearch OU=users,DC=mydomain,DC=com

# LDAP search base

# default is empty line

#LDAPBaseSearch ou=users,dc=mydomain,dc=local

# TAG: LDAPFilterSearch uid=%s

# User search filter by user's logins in LDAP

# First founded record will be used

# %s - will be changed to userlogins from access.log file

# filter string can have some tags '%s'

# default value is 'uid=%s'

#LDAPFilterSearch uid=%s

# TAG: LDAPTargetAttr attributename

# Name of the attribute containing a name of the user

# default value is 'cn'

#LDAPTargetAttr cn

# TAG: long_url yes|no

# If yes, the full url is showed in report.

# If no, only the site will be showed

# YES option generate very big sort files and reports.

long_url no

# TAG: date_time_by bytes|elap

# Date/Time reports show the downloaded volume or the elapsed time or both.

date_time_by bytes

# TAG: charset name

# ISO 8859 is a full series of 10 standardized multilingual single-byte coded (8bit)

# graphic character sets for writing in alphabetic languages

# You can use the following charsets:

# Latin1 - West European

# Latin2 - East European

# Latin3 - South European

# Latin4 - North European

# Cyrillic
# Arabic

# Greek

# Hebrew

# Latin5 - Turkish

# Latin6

# Windows-1251

# Japan

# Koi8-r

# UTF-8

charset Latin1

# TAG: user_invalid_char "&/"

# Records that contain invalid characters in userid will be ignored by Sarg.

#user_invalid_char "&/"

# TAG: privacy yes|no

# privacy_string "***.***.***.***"

# privacy_string_color blue

# In some countries the sysadm cannot see the visited sites by a restrictive law.

# Using privacy yes the visited url will be changes by privacy_string and the link

# will be removed from reports.

#privacy no

#privacy_string "***.***.***.***"

#privacy_string_color blue

# TAG: include_users "user1:user2:...:usern"

# Reports will be generated only for listed users.

#include_users none

# TAG: exclude_string "string1:string2:...:stringn"

# Records from access.log file that contain one of listed strings will be ignored.

#exclude_string none

# TAG: show_successful_message yes|no

# Shows "Successful report generated on dir" at end of process.

show_successful_message no

# TAG: show_read_statistics yes|no

# Shows some reading statistics.

show_read_statistics no

# TAG: topuser_fields

# Which fields must be in Topuser report.

topuser_fields NUM DATE_TIME USERID CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE

# TAG: user_report_fields

# Which fields must be in User report.

user_report_fields CONNECT BYTES %BYTES IN-CACHE-OUT USED_TIME MILISEC %TIME TOTAL AVERAGE

# TAG: bytes_in_sites_users_report yes|no

# Bytes field must be in Site & Users Report ?

#bytes_in_sites_users_report no

# TAG: topuser_num n

# How many users in topsites report. 0 = no limit

topuser_num 0

# TAG: datafile file

# Save the report results in a file to populate some database

#datafile none

# TAG: datafile_delimiter ";"

# ascii character to use as a field separator in datafile

#datafile_delimiter ";"

# TAG: datafile_fields all

# Which data fields must be in datafile

# user;date;time;url;connect;bytes;in_cache;out_cache;elapsed

#datafile_fields user;date;time;url;connect;bytes;in_cache;out_cache;elapsed

# TAG: datafile_url ip|name

# Saves the URL as ip or name in datafile

#datafile ip

# TAG: weekdays

# The weekdays to take account ( Sunday->0, Saturday->6 )

# Example:

#weekdays 1-3,5

# Default:

#weekdays 0-6

# TAG: hours

# The hours to take account

# Example:

#hours 7-12,14,16,18-20

# Default:

#hours 0-23

# TAG: dansguardian_conf file

# DansGuardian.conf file path

# Generate reports from DansGuardian logs.

# Use 'none' to disable it.

# dansguardian_conf /usr/dansguardian/dansguardian.conf

#dansguardian_conf none

# TAG: dansguardian_filter_out_date on|off

# This option replaces dansguardian_ignore_date whose name was not appropriate with respect to its action.

# Note the change of parameter value compared with the old option.

# 'off' use the record even if its date is outside of the range found in the input log file.

# 'on' use the record only if its date is in the range found in the input log file.

#dansguardian_filter_out_date on

# TAG: squidguard_conf file

# path to squidGuard.conf file

# Generate reports from SquidGuard logs.

# Use 'none' to disable.

# You can use sarg -L filename to use an alternate squidGuard log.

# squidguard_conf /usr/local/squidGuard/squidGuard.conf

#squidguard_conf none

# TAG: redirector_log file

# the location of the web proxy redirector log such as one created by squidGuard or Rejik. The option

# may be repeated up to 64 times to read multiple files.

# If this option is specified, it takes precedence over squidguard_conf.

# The command line option -L override this option.

#redirector_log /usr/local/squidGuard/var/logs/urls.log

# TAG: redirector_filter_out_date on|off

# This option replaces squidguard_ignore_date and redirector_ignore_date whose names were not

# appropriate with respect to their action.

# Note the change of parameter value compared with the old options.

# 'off' use the record even if its date is outside of the range found in the input log file.

# 'on' use the record only if its date is in the range found in the input log file.

#redirector_filter_out_date on

# TAG: redirector_log_format

# Format string for web proxy redirector logs.

# This option was named squidguard_log_format before sarg 2.3.

# REJIK #year#-#mon#-#day# #hour# #list#:#tmp# #ip# #user# #tmp#/#tmp#/#url#/#end#

# SQUIDGUARD #year#-#mon#-#day# #hour# #tmp#/#list#/#tmp#/#tmp#/#url#/#tmp# #ip#/#tmp# #user# #end#

#redirector_log_format #year#-#mon#-#day# #hour# #tmp#/#list#/#tmp#/#tmp#/#url#/#tmp# #ip#/#tmp# #user# #end#

# TAG: show_sarg_info yes|no

# shows sarg information and site path on each report bottom

#show_sarg_info yes

# TAG: show_sarg_logo yes|no

# shows sarg logo

#show_sarg_logo yes

# TAG: parsed_output_log directory

# Saves the processed log in a sarg format after parsing the squid log file.

# This is a way to dump all of the data structures out, after parsing from

# the logs (presumably this data will be much smaller than the log files themselves),

# and pull them back in for later processing and merging with data from previous logs.

#parsed_output_log none

# TAG: parsed_output_log_compress /bin/gzip|/usr/bin/bzip2|nocompress

# Command to run to compress sarg parsed output log. It may contain

# options (such as -f to overwrite existing target file). The name of

# the file to compresse is provided at the end of this

# command line. Don't forget to quote things appropriately.

#parsed_output_log_compress /bin/gzip

# TAG: displayed_values bytes|abbreviation

# how the values will be displayed in reports.

# eg. bytes - 209.526

# abbreviation - 210K

#displayed_values bytes

# Report limits

# TAG: authfail_report_limit n

# TAG: denied_report_limit n

# TAG: siteusers_report_limit n

# TAG: squidguard_report_limit n

# TAG: user_report_limit n

# TAG: dansguardian_report_limit n

# TAG: download_report_limit n

# report limits (lines).

# '0' no limit

#authfail_report_limit 10

#denied_report_limit 10

#siteusers_report_limit 0

#squidguard_report_limit 10

#dansguardian_report_limit 10

#user_report_limit 10

#user_report_limit 50

# TAG: www_document_root dir

# Where is your Web DocumentRoot

# Sarg will create sarg-php directory with some PHP modules:

# - sarg-squidguard-block.php - add urls from user reports to squidGuard DB

#www_document_root /var/www/html

# TAG: block_it module_url

# This tag allow you to pass urls from user reports to a cgi or php module,

# to be blocked by some Squid acl

# Eg.: block_it /sarg-php/sarg-block-it.php

# sarg-block-it is a php that will append a url to a flat file.

# You must change /var/www/html/sarg-php/sarg-block-it to point to your file

# in $filename variable, and chown to a httpd owner.

# sarg will pass http://module_url?url=url

#block_it none

# TAG: external_css_file path

# Provide the path to an external css file to link into the HTML reports instead of

# the inline css written by sarg when this option is not set.

# In versions prior to 2.3, this used to be an absolute file name to

# a file to include verbatim in each HTML page but, as it takes a lot of

# space, version 2.3 switched to a link to an external css file.

# Therefore, this option must contain the HTTP server path on which a client

# browser may find the css file.

# Sarg use theses style classes:

# .logo logo class

# .info sarg information class, align=center

# .title_c title class, align=center

# .header_c header class, align:center

# .header_l header class, align:left

# .header_r header class, align:right

# .text text class, align:right

# .data table text class, align:right

# .data2 table text class, align:left

# .data3 table text class, align:center

# .link link class

# Sarg can be instructed to output the internal css it inline

# into the reports with this command:

# sarg --css

# You can redirect the output to a file of your choice and edit

# it to your liking.

#external_css_file none

# TAG: user_authentication yes|no

# Allow user authentication in User Reports using .htaccess

# Parameters:

# AuthUserTemplateFile - The template to use to create the

# .htaccess file. In the template, %u is replaced by the

# user's ID for which the report is generated. The path of the

# template is relative to the directory containing sarg

# configuration file.

# user_authentication no

# AuthUserTemplateFile sarg_htaccess

# TAG: download_suffix "suffix,suffix,...,suffix"

# file suffix to be considered as "download" in Download report.

# Use 'none' to disable.

download_suffix "zip,arj,bzip,gz,ace,doc,iso,adt,bin,cab,com,dot,drv$,lha,lzh,mdb,mso,ppt,rtf,src,shs,sys,exe,dll,mp3,avi,mpg,mpeg"

# TAG: ulimit n

# The maximum number of open file descriptors to avoid "Too many open files" error message.

# You need to run sarg as root to use ulimit tag.

# If you run sarg with a low privilege user, set to 'none' to disable ulimit

#ulimit 20000

# TAG: ntlm_user_format username|domainname+username

# NTLM users format.

#ntlm_user_format domainname+username

# TAG: realtime_refresh_time num sec

# How many time to auto refresh the realtime report

# 0 = disable

# realtime_refresh_time 3

# TAG: realtime_access_log_lines num

# How many last lines to get from access.log file

# realtime_access_log_lines 1000

# TAG: realtime_types: GET,PUT,CONNECT,ICP_QUERY,POST

# Which records must be in realtime report.

# realtime_types GET,PUT,CONNECT

# TAG: realtime_unauthenticated_records: ignore|show

# What to do with unauthenticated records in realtime report.

# realtime_unauthenticated_records: show

# TAG: byte_cost value no_cost_limit

# Cost per byte.

# Eg. byte_cost 0.01 100000000

# per byte cost = 0.01

# bytes with no cost = 100 Mb

# 0 = disable

# byte_cost 0.01 50000000

# TAG: squid24 on|off

# Compatilibity with squid version <= 2.4 when using emulate_http_log on

# squid24 off

mkdir /var/www/squid-reports

Nano /etc/sarg/sarg-reports.conf

root@raabtaproxy:/etc/sarg# cat sarg-reports.conf

SARG=/usr/bin/sarg

CONFIG=/etc/sarg/sarg.conf

HTMLOUT=/var/www/squid-reports

PAGETITLE="Access Reports on $(hostname)"

LOGOIMG=/sarg/images/sarg.png

LOGOLINK="http://$(hostname)/"

DAILY=Daily

WEEKLY=Weekly

MONTHLY=Monthly

EXCLUDELOG1="SARG: No records found"

EXCLUDELOG2="SARG: End"

## To generate report of today
/usr/sbin/sarg-reports today

Crontab to generate reports
root#crontab -e
00 00 * * * /usr/sbin/sarg-reports daily
00 01 * * 1 /usr/sbin/sarg-reports weekly
03 02 1 * * /usr/sbin/sarg-reports monthly

Brain Book

Tuesday, 23 April 2013

FIX BAD SECTORS IN HARD DRIVE

Friday, 19 April 2013

SQUID 3.3.3 COMPILATION ON UBUNTU 12

Wednesday, 17 April 2013

ZeroShell-2.0.RC2 Installation on Hard Disk

Wednesday, 3 April 2013

HOW TO SHUTDOWN AND RESTART LINUX

How do I use shutdown command?

How do I shutdown computer at specific time?

How do I reboot computer?

SARG INSTALLATION ON UBUNTU 12