Installing Buffer optimised code on Webdav aliased hosts

Currently the Webdav optmised code is build as a set of library files (rather than packages as an RPM), and updated xrootd cfg files.

Latest configuration

Current / testing

33dfdd3_2 ; latest version running on the gateways; no known issues (aside from the generally observed stability problems).

Library files

The necessary library files are:

libXrdCeph-5.so
libXrdCephPosix.so
libXrdCephPosix.so.0
libXrdCephPosix.so.0.0.1
libXrdCephXattr-5.so

And are put into their own directory, so not to overwrite the standard files.

Installation

Below COMMIT refers to a directory containing the required library files for XrdCeph. While it doesn't have to refer to the commit version, it's probably sensible that it does.

COMMIT=33dfdd3_2
cd ~/bufferCode/${COMMIT}
mkdir /usr/local/lib64/xrootd_buffer
chown xrootd:xrootd -R  /usr/local/lib64/xrootd_buffer
cd ..
cp -r ${COMMIT} /usr/local/lib64/xrootd_buffer/.


systemctl edit xrootd@proxy.service --full
# change line to this:   LimitNOFILE=655360

cd /etc/xrootd
# udpate the xrootd-proxy.cfg and xrootd-ceph.cfg files

systemctl daemon-reload
systemctl restart xrootd@proxy && systemctl restart xrootd@ceph

# noquattor the host

echo "WebDav Optimisations" > /etc/noquattor

xrootd-proxy

Example of the proxy config file; not there are host-specific lines

# The export directive indicates which paths are to be exported. While the
# default is '/tmp', we indicate it anyway to show you this directive.
#
all.export *?
all.export /

# The adminpath and pidpath variables indicate where the pid and various
# IPC files should be placed
#
all.adminpath /var/spool/xrootd
all.pidpath /var/run/xrootd
#xrootd.async segsize 67108864
xrootd.async off
xrd.buffers maxbsz 67108864

# Configure TPC
# ensure cksum adler32 is included in the tpc directive, in order to caclulate by default on transfer
ofs.tpc cksum adler32 fcreds ?gsi =X509_USER_PROXY autorm xfr 40 pgm /etc/xrootd/xrdcp-tpc.sh

# Configure proxy memory caching
#
ofs.osslib /usr/lib64/libXrdPss.so
ofs.ckslib * /usr/lib64/libXrdPss.so
pss.origin ceph-svc97.gridpp.rl.ac.uk:1095
#pss.cache max2cache 16777216 pagesize 16777216 size 17179869184

# add this line to trigger external checksum calculation. Would be overwritten by other xrootd.chksum lines
xrootd.chksum max 50 adler32 /etc/xrootd/xrd_cephsum.sh

#xrd.trace all

# Configure gsi security
#
xrootd.seclib /usr/lib64/libXrdSec.so
sec.protparm gsi -vomsfun:/usr/lib64/libXrdSecgsiVOMS.so -vomsfunparms:certfmt=pem|vos=atlas,dteam|grps=/atlas,/dteam|grpopt=10|dbg
sec.protocol unix
sec.protocol gsi -dlgpxy:1 -exppxy:=creds -crl:3 -cert:/etc/grid-security/xrootd/hostcert.pem -key:/etc/grid-security/xrootd/hostkey.pem -gridmap:/etc/grid-security/grid-mapfile -gmapopt:2 -gmapto:3600 -d:0
sec.protbind * only gsi
sec.protbind castor-ns-snapshot.gridpp.rl.ac.uk unix

# Configure authorisation
#
ofs.authorize
acc.authdb /etc/grid-security/authdb
acc.audit deny grant

# Name-to-name mapping
#

pss.namelib /usr/lib64/libXrdCmsTfc.so file:/etc/xrootd/storage.xml?protocol=xrootd,https,http,davs


# Configure internal fstream event reporting
# details: https://wiki.e-science.cclrc.ac.uk/web1/bin/view/EScienceInternal/XrootdMonitoring
#
xrootd.monitor all auth fstat 10s ops lfn xfr 1 ident 1m dest fstat info user redir 172.16.105.115:9931

if exec xrootd
  # both xrootd and https will use port 1094 !
  xrd.protocol http libXrdHttp.so
fi



# Dont use voms but rather the grid-mapfile:
# http.secxtractor /usr/lib64/libXrdSecgsiVOMS.so
http.gridmap /etc/grid-security/grid-mapfile


# /etc/xrootd/robots.txt
# it is used to discourage Google search of http(s)
# User-agent: *
# Disallow: /
http.staticpreload http://static/robots.txt /etc/xrootd/robots.txt


# Require the use of the xrd.tls certificates (alternative is to use manual)
http.httpsmode auto


# HTTP TPC, see https://twiki.cern.ch/twiki/bin/view/Main/XRootDoverHTTP#Enable_Third_Party_Copy
http.exthandler xrdtpc libXrdHttpTPC.so
http.header2cgi Authorization authz

# Please install libmacaroons rpm from EPEL.
# Macaroons support, see: https://twiki.cern.ch/twiki/bin/view/Main/XRootDoverHTTP#Macaroons_Support
http.exthandler xrdmacaroons libXrdMacaroons.so


# the secret must be the same on all external gateways
# the current secret can be found on keys.gridpp in /var/keys/echo-xrootd-token/, 
# and should be copied into the location below, owned by xrootd:xrootd (perms 440)
# original command used to generate: 
# openssl rand -base64 -out /etc/xrootd/macaroon-secret 64 (make single line)
macaroons.secretkey /etc/xrootd/macaroon-secret
ofs.authlib libXrdMacaroons.so

# TLS
xrd.tls /etc/grid-security/xrootd/hostcert.pem /etc/grid-security/xrootd/hostkey.pem
xrd.tlsca certdir /etc/grid-security/certificates
#xrootd.tls capable all -data
xrootd.tls off 

all.sitename T1_UK_RAL

xrootd-ceph

Example of the ceph config file; note there might be host-specific lines

###########################################################################
# This is a very simple sample configuration file sufficient to start an  #
# xrootd data server using the default port 1094. This server runs by     #
# itself (stand-alone) and does not assume it is part of a cluster. You   #
# can then connect to this server to access files in '/tmp'.              #
# Consult the the reference manuals on how to create more complicated     #
# configurations.                                                         #
#                                                                         #
# On successful start-up you will see 'initialization completed' in the   #
# last message. You can now connect to the xrootd server.                 #
#                                                                         #
# Note: You should always create a *single* configuration file for all    #
# daemons related to xrootd.                                              #
###########################################################################

# The export directive indicates which paths are to be exported. While the
# default is '/tmp', we indicate it anyway to show you this directive.
#
all.export *?
all.export /

# The adminpath and pidpath variables indicate where the pid and various
# IPC files should be placed
#
all.adminpath /var/spool/xrootd
all.pidpath /var/run/xrootd
#xrootd.async segsize 67108864
xrootd.async  off
xrd.buffers maxbsz 67108864

# Configure sss security
#
xrootd.seclib /usr/lib64/libXrdSec.so
sec.protocol sss -s /etc/grid-security/xrootd/sss.keytab.grp -c /etc/grid-security/xrootd/sss.keytab.grp
sec.protbind * only sss

# Configure rados connection
#
#ofs.osslib +cksio /usr/lib64/libXrdCeph.so xrootd@,1,8388608,67108864
#ofs.xattrlib /usr/lib64/libXrdCephXattr.so
ofs.osslib +cksio /usr/local/lib64/xrootd_buffer/33dfdd3_2/libXrdCeph.so xrootd@,1,8388608,67108864
ofs.xattrlib /usr/local/lib64/xrootd_buffer/33dfdd3_2/libXrdCephXattr.so
xrootd.chksum max 50 adler32 /etc/xrootd/xrd_cephsum.sh

# Configure the port
#
xrd.port 1095

# 5 is good
ceph.nbconnections 10
ceph.usebuffer 1
#ceph.usebuffer 1
#ceph.buffersize 1048576
#ceph.buffersize 2097152
#ceph.buffersize 4194304
#ceph.buffersize 8388608
ceph.buffersize 16777216
#ceph.buffersize 33554432
#ceph.buffersize 67108864
#ceph.buffersize 134217728
ceph.usereadvalg 0
ceph.readvalgname passthrough
#ceph.readvalgname basic

XRootD

Non aqualon-ed configuration setup on davs gateways config