Friday, October 9, 2015

mount s3 on linux

Steps:
# yum remove fuse fuse-s3fs
# yum install gcc libstdc++-devel gcc-c++ curl-devel libxml2-devel openssl-devel mailcap
# wget http://downloads.sourceforge.net/project/fuse/fuse-2.X/2.9.3/fuse-2.9.3.tar.gz
# tar xvfs fuse-2.9.3.tar.gz
# cd fuse-2.9.3
# ./configure --prefix=/usr/local
# make
# make install
# yum install fuse-libs

# export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
# ldconfig
# modprobe fuse

# wget https://s3fs.googlecode.com/files/s3fs-1.74.tar.gz
# tar -xvf s3fs-1.74.tar.gz
# cd s3fs-1.74
# ./configure --prefix=/usr/local
# make
# make install

[root]# mkdir /s3logs

At this point create a new user in AWS IAM, download the accesskey and secret key.
Make sure you grant this user the access to S3 bucket.


[root@pgvmdc s3]# cat s3_access_key
AWSAccessKeyId=XXXXXXXXXXXXXXXX
AWSSecretKey=YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY

[root@pgvmdc s3]# echo XXXXXXXXXXXXXXXXXXX:YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY  > ~/.passwd-s3fs

[root@pgvmdc s3]# chmod 600 /root/.passwd-s3fs

[root@pgvmdc s3]# cat ~/.passwd-s3fs
XXXXXXXXXXXXXXXXXXX:YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY

[root@pgvmdc s3]# mkdir /tmp/cache
[root@pgvmdc s3]# chmod 777 /tmp/cache/
[root@pgvmdc s3]# chmod 777 /s3mnt/



[root]# s3fs -o use_cache=/tmp/cache logserver  /s3logs
fuse: warning: library too old, some operations may not not work

[root@pgvmdc s3]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/xvda2       25G  7.8G   16G  34% /
tmpfs           497M     0  497M   0% /dev/shm
/dev/xvda1      248M   82M  154M  35% /boot
s3fs            256T     0  256T   0% /s3logs


Add this to fstab
/etc/fstab
s3fs#logserver /s3logs fuse allow_other,use_cache=/tmp/cache 0 0


Errors:
##1
[root]# s3fs -o use_cache=/tmp/cache log-server-bkup /s3logs/
s3fs: could not determine how to establish security credentials

## resolution ##
make sure the .passwd-s3fs is in the home directory

##2
[root]# s3fs -o use_cache=/tmp/cache log-server-bkup /s3logs/
s3fs: error while loading shared libraries: libfuse.so.2: cannot open shared object file: No such file or directory

## resolution ##
make sure you have installed fuse-libs
[root]# yum install fuse-libs


#3. # s3fs -o use_cache=/tmp/cache log-server-bkup  /s3logs
s3fs: unable to access MOUNTPOINT /s3logs: Transport endpoint is not connected

## resolution ## (This typically is caused by the mount directory being left mounted due to a crash of your filesystem)
#mv /s3logs /tmp
#mkdir /s3logs
mount again