big data task

[Q-1]
> db.user.find()
{ "_id" : ObjectId("59ba31772515ce6bd280eccd"), "Userid" : 1, "Data_of_join" : "sep 8,2016", "Education" : [ "BCA", "MCA" ], "Hobby" : [ "paying songs", "music" ], "job_category" : "manager", "Gender" : "male" }
{ "_id" : ObjectId("59ba317d2515ce6bd280ecce"), "Userid" : 2, "Data_of_join" : "jan 08,2014", "Education" : [ "BCA", "MCA" ], "Hobby" : [ "paying songs", "music" ], "job_category" : "analyst", "Gender" : "female" }
{ "_id" : ObjectId("59ba31802515ce6bd280eccf"), "Userid" : 3, "Data_of_join" : "march 07,2009", "Education" : [ "BCA", "MCA" ], "Hobby" : [ "paying songs", "music" ], "job_category" : "coder", "Gender" : "female" }
{ "_id" : ObjectId("59ba31bb2515ce6bd280ecd0"), "Userid" : 4, "Data_of_join" : "fab 10,2011", "Education" : [ "BCA", "MCA" ], "Hobby" : [ "paying songs", "music" ], "job_category" : "designer", "Gender" : "male" }
{ "_id" : ObjectId("59ba31e52515ce6bd280ecd1"), "Userid" : 5, "Data_of_join" : "march 07,2009", "Education" : [ "BCA", "MCA" ], "Hobby" : [ "paying songs", "music" ], "job_category" : "tester", "Gender" : "female" }
> db.user.count({Hobby:"paying songs"})
[Q-2]
>db.Movies.find()
{ "_id" : ObjectId("59ba3be92515ce6bd280ecd2"), "movieId" : 1, "movieName" : "wanted", "actors" : [ { "actorName" : "salman", "gender" : "male", "roleName" : "Hero" } ], "releaseYear" : 2010 }
{ "_id" : ObjectId("59ba3d5c2515ce6bd280ecd3"), "movieId" : 1, "movieName" : "wanted", "actors" : [ { "reviewID" : "10", "reviewDate" : "sep 08 2010", "reviewerName" : "ravi", "reviewRating" : "3 star", "reviwComment" : "NiceMovie" } ] }
{ "_id" : ObjectId("59ba3dcd2515ce6bd280ecd4"), "movieId" : 2, "movieName" : "veer", "actors" : [ { "reviewID" : "11", "reviewDate" : "sep 10 2011", "reviewerName" : "ravi", "reviewRating" : "5 star", "reviwComment" : "NiceMovie" } ] }
{ "_id" : ObjectId("59ba3ec22515ce6bd280ecd5"), "movieId" : 3, "movieName" : "razz", "actors" : [ { "reviewID" : "9", "reviewDate" : "jan 7 2015", "reviewerName" : "ravi", "reviewRating" : "2 star", "reviwComment" : "NiceMovie" } ] }
{ "_id" : ObjectId("59ba3efd2515ce6bd280ecd6"), "movieId" : 4, "movieName" : "tum", "actors" : [ { "reviewID" : "8", "reviewDate" : "fab 05 2013", "reviewerName" : "ravi", "reviewRating" : "1 star", "reviwComment" : "NiceMovie" } ] }
>

Mongo commands

Mongo commands

v  The showing Database

> show dbs

admin         0.000GB

emp           0.000GB

jay           0.000GB

job           0.000GB

local         0.000GB

movieratings  0.000GB

mydb          0.000GB

test          0.000GB

> use Mydb_Rakesh

switched to db Mydb_Rakesh

v  showing the Tables

> show collections

people

> db.createCollection("people");

{ "ok" : 1 }

>

v  Insert Record On Tables

> db.people.insert(

... {_id:1,

... Name:"Rakesh",

... City:"Rajkot"

... }

...  );

WriteResult({ "nInserted" : 1 })

>

> db.people.insert( {_id:2, Name:"Mahesh", City:"Div" }  );

WriteResult({ "nInserted" : 1 })

> db.people.insert( {_id:3, Name:"Jayesh", City:"Ahemdabad" }  );

WriteResult({ "nInserted" : 1 })

> db.people.insert( {_id:4, Name:"Prakash", City:"Surat" }  );

WriteResult({ "nInserted" : 1 })

> db.people.insert( {_id:5, Name:"Yash", City:"Junagadh" }  );

WriteResult({ "nInserted" : 1 })

                                   

v  Update Records

> db.people.update( {Name:"Yash"},{$set:{City:"Kesod"}}  );

WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })

v  Delete Records

> db.people.deleteMany( {_id:4}  );

{ "acknowledged" : true, "deletedCount" : 1 }

v  Select Records

> db.people.find( );

{ "_id" : 1, "Name" : "Rakesh", "City" : "Rajkot" }

{ "_id" : 2, "Name" : "Mahesh", "City" : "Div" }

{ "_id" : 3, "Name" : "Jayesh", "City" : "Ahemdabad" }

{ "_id" : 5, "Name" : "Yash", "City" : "Kesod" }

>

relational operators:

1	$eq	Equal to
2	$ne	Not Equal to
3	$gte	Greater than or equal to
4	$lte	Less than or equal to
5	$gt	Greater than
6	$lt	Less than
7	$in	Inside the field
8	$nin	Not inside the field

> db.people.find( );

{ "_id" : 1, "Name" : "Rakesh", "City" : "Rajkot" }

{ "_id" : 2, "Name" : "Mahesh", "City" : "Div" }

{ "_id" : 3, "Name" : "Jayesh", "City" : "Ahemdabad" }

{ "_id" : 5, "Name" : "Yash", "City" : "Kesod" }

> db.people1.insert({user_id:"m001",age:25,status:"A"});

WriteResult({ "nInserted" : 1 })

> db.people1.insert({user_id:"m002",age:22,status:"B"});

WriteResult({ "nInserted" : 1 })

> db.people1.insert({user_id:"m003",age:20,status:"C"});

WriteResult({ "nInserted" : 1 })

> db.people1.insert({user_id:"m004",age:25,status:"C"});

WriteResult({ "nInserted" : 1 })

> db.people1.insert({user_id:"m005",age:30,status:"A"});

WriteResult({ "nInserted" : 1 })

> db.people1.find();

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "age" : 22, "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "A" }

> db.people1.updateMany({age:{$gt:25}},{$set:{status:"C"}});

{ "acknowledged" : true, "matchedCount" : 1, "modifiedCount" : 1 }

> db.people1.find();

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "age" : 22, "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

> db.people1.find({},{user_id:"m001",status:"1"});

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "status" : "A" }

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "status" : "C" }

> db.people2.find({status:{$ne:"A"}});

> db.people1.find({status:{$ne:"A"}});

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "age" : 22, "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

> db.people1.find({age:{$gt:22}});

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

> db.people1.find({age:{$lt:22}});

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

> db.people1.find({user_id:/m004/});

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

> db.people1.find({user_id:/^m/});

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "age" : 22, "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

> db.people1.find({status:"A"}).sort({user_id:1})

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

> db.people1.find({status:"A"}).sort({user_id:-1})

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

> db.people1.find().count();

5

> db.people1.find({user_id:{$exists:true}});

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

{ "_id" : ObjectId("59c0df6d2ad20fc1c30053c2"), "user_id" : "m002", "age" : 22, "status" : "B" }

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

> db.people1.find({user_id:{$exists:true}}).count();

5

> db.people1.distinct("status");

[ "A", "B", "C" ]

> db.people1.distinct("age");

[ 25, 22, 20, 30 ]

> db.people1.distinct("user_id");

[ "m001", "m002", "m003", "m004", "m005" ]

> db.people1.find().limit(1);

{ "_id" : ObjectId("59c0df542ad20fc1c30053c1"), "user_id" : "m001", "age" : 25, "status" : "A" }

> db.people1.find().limit(4).skip(10);

> db.people1.find().limit(4).skip(2);

{ "_id" : ObjectId("59c0df7a2ad20fc1c30053c3"), "user_id" : "m003", "age" : 20, "status" : "C" }

{ "_id" : ObjectId("59c0df8e2ad20fc1c30053c4"), "user_id" : "m004", "age" : 25, "status" : "C" }

{ "_id" : ObjectId("59c0df9d2ad20fc1c30053c5"), "user_id" : "m005", "age" : 30, "status" : "C" }

>

Subject Name: Big Data Tools (BDT) Subject Code: 3650010

-> Local file system                                                               

-> HDFS                                                                                                        

-> abc.txt    (Both Are Diff File)

-> abc.txt                                                                                                                                            

-> ls    

-> -copyFromLocal (Upload hadoop)                                                                        

              (Download in loacl)

            -> -moveFromLocal

           -> -moveToLocal

           -ls

            -> -mkdir

            -> -put (Upload)

            -> -get (Download)

            -> -du (Disk Usages) -> Byte

            -> -dus(Disk Usages Summary total size of particular file)

// Virtual Disk Image Path :-

/data/hworkspace/Hortonworks Sandbox with HDP 2.3.2/Hortonworks Sandbox with HDP 2.3.2-disk1.vmdk #

Login Old

-----------

User name :- root

Password :- hadoop

Login New

--------------

User name :- root

Password :- mca@123

==================

In Terminal:-

==================

1)         ssh root@127.0.0.1 -p 2222

2)         mca@123

            su hdfs  ( su = Super User)

            exit

----------

Commands

----------

=>        chmod 777 /opt

=>        cd /opt

=>        mkdir

=>       hadoop fs         ( use for Help old )

=>       hdfs dfs           ( use for Help new )

=>       hadoop namenode -formate (formate the drive(namenode))

hadoop fs -ls

hadoop fs -mkdir Abc

hadoop fs -ls

hdfs dfs -mkdir help

hdfs dfs -ls

MongoDB

1)  ssh root@127.0.0.1 -p 2222

2)  mca@123

3)  mongo

ü  Insert

db.emp.insert({ _id:1 , ENAME:"Scott" , SAL:5000 });

db.emp.insert({ _id:2 , ENAME:"King"  , SAL:1000 });

db.emp.insert({ _id:3 , ENAME:"Peter" , SAL:2000 });

db.emp.insert({ _id:4 , ENAME:"Smith" , SAL:4000 });

db.emp.insert({ _id:5 , ENAME:"John"  , SAL:3000 });

ü  Update  

Parameter :=

(           $set : document[column]        ,           upsert : true/false         ,           multi : true/false          )

db.emp.update({ _id:5 } , { $set: { SAL:3500 } });

db.emp.update({ _id:2 } , { $set: { ENAME:"King" , SAL:7000 } });

db.emp.update({ _id:5 } , { ENAME:"AG" } , { upsert : true });

db.emp.update({ _id:5 } , { $set:{ SAL:3000 } } , { upsert:true });

db.emp.update({ SAL:{ $gt:2000 } } , { $set: { ENAME:"Vijay" } }, { multi:true });

db.emp.update({ _id:5 } , { $set: { BDATE:29 , NAME:"AG" } } , { upsert:true });

ü  Deleted using Update

Parameter :=

(           $unset : { document: 1 }         )

db.emp.update({ _id:5 } , { $unset: { BDATE:1 }});

ü  Find

1 or true

0 or false

db.emp.find( { SAL: { $gt: 3000 } } )

db.emp.find( { SAL: { $gt: 2000, $lt: 5001 } } );

db.emp.find( { SAL: { $gt: 2000, $lt: 5001 } } , { _id:0 , ENAME:1 } );