2023. 6. 30. 09:35ㆍAWS 기반 데이터분석 처리 고급 SW 클라우드 개발자 양성과정
Day-66
# ssh master
# apt -y update; apt -y upgrade
# master, worker1, worker2 EC2 -> 인스턴스 재부팅
# ssh awsu
# mkdir -p /allnew/node/module
### http request - GET
## request-get.js
const https = require('https');
const options = {
hostname : 'example.com',
port : 3000,
path : '/todos',
method : 'GET'
}
const req = https.request(options, res => {
console.log(`statusCode : ${res.statusCode}`)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.err(error)
})
req.end()
### http request - POST
## request-post.js
const https = require('https');
const data = JSON.stringify({
todo: 'Buy the milk'
})
const options = {
hostname : 'example.com',
port : 3000,
path : '/todos',
method : 'POST',
header : {
'Content-Type' : 'application/json',
'Content-Length' : data.length,
}
}
const req = https.request(options, res => {
console.log(`statusCode : ${res.statusCode}`)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.err(error)
})
req.write(data)
req.end()
### http request - axios
## request-axios.js
const axios = require('axios');
axios
.post('https://example.com/todos',{
todo : "Buy the milk"
})
.then(res => {
console.log(`statusCode : ${res.statusCode}`)
console.log(res)
})
.catch(error => {
console.error(error)
})
### pretty json print nodejs module
## pretty-json.js
const prettier = require('prettier’);
…..
res.json(prettier.format(JSON.stringify(object)));
### pandas nodejs module
## pandas.js
const pd = require('node-pandas’);
…..
df = pd.DataFrame(res);
df.show;
### danfojs-node nodejs module
## danfojs-node.js
const dfd = require('danfojs-node');
…
df = new dfd.DataFrame(res);
df.print();
### multer module
# cd ..
# cp -r s3 multer
# cd multer
## app.js
const express = require('express');
const morgan = require('morgan');
const path = require('path');
const fs = require("fs")
const bodyParser = require('body-parser');
const cookieParser = require('cookie-parser');
const app = express();
app.set('port', process.env.PORT || 8000);
app.set('views', path.join(__dirname, 'public'));
app.set('view engine','ejs');
app.use(morgan('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
var main = require('./routes/main.js');
app.use('/', main);
app.listen(app.get('port'), () => {
var dir = './uploadedFiles';
if (!fs.existsSync(dir)) fs.mkdirSync(dir);
console.log('8000 Port : Server Started...')
});
## public/upload.ejs
<h1><i>Express</i> and <i>Multer</i> File Upload Example </h1>
<hr>
<h3>1. Single File upload</h3>
<form action="/uploadFile" enctype="multipart/form-data" method="post">
<input type="file" name="attachment">
<button type="submit" class="btn btn-primary">Upload</button>
</form>
<hr>
<h3>2. Single File upload (Keep original Filename)</h3>
<form action="/uploadFileWithOriginalFilename" enctype="multipart/form-data" method="post">
<input type="file" name="attachment">
<button type="submit" class="btn btn-primary">Upload</button>
</form>
<hr>
<h3>3. Multi File upload</h3>
<form action="/uploadFiles" enctype="multipart/form-data" method="post">
<input type="file" multiple name="attachments">
<button type="submit" class="btn btn-primary">Upload</button>
</form>
<hr>
<h3>4. Multi File upload (Keep original Filename)</h3>
<form action="/uploadFilesWithOriginalFilename" enctype="multipart/form-data" method="post">
<input type="file" multiple name="attachments">
<button type="submit" class="btn btn-primary">Upload</button>
</form>
<hr>
<iframe name=targetURL class="iframe-preview center" width="90%" height="200" style="border:none;" src="list">
</iframe>
## public/confirmation.ejs (파일 성공했을떄 창 나타내느 문구)
<h1>Success!</h1>
<% if(file) { %>
<pre><%=JSON.stringify(file, null, 2) %></pre>
<% } %>
<% if(files) { %>
<pre><%=JSON.stringify(files, null, 2) %></pre>
<% } %>
<a href='/'>Back</a>
## routes/main.js
const express = require('express');
const app = express.Router();
const multer = require("multer")
const fs = require('fs')
var storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, 'uploadedFiles/');
},
filename(req, file, cb) {
cb(null, `${Date.now()}__${file.originalname}`);
},
});
var upload = multer({ dest : 'uploadedFiles/' });
var uploadWithOriginalFilename = multer({ storage : storage });
const fullPath = process.cwd() + '/uploadedFiles' //(not __dirname) ### fullPath: 현재 경로
app.get('/', function(req, res) {
res.render('upload');
});
app.post('/uploadFile', upload.single('attachment'), function (req, res) { ### File upload 성공하면
res.render('confirmation', { file:req.file, files:null }); ### Confirmation에서 작동
});
app.post('/uploadFileWithOriginalFilename', uploadWithOriginalFilename.single('attachment'), function (req, res) {
res.render('confirmation', { file:req.file, files:null }); ### SIngle FIle
});
app.post('/uploadFiles', upload.array('attachments'), function (req, res) {
res.render('confirmation', { file:null, files:req.files }); ### Multiple File
});
app.post('/uploadFilesWithOriginalFilename', uploadWithOriginalFilename.array('attachments'), function (req, res) {
res.render('confirmation', { file:null, files:req.files });
});
app.get('/list', (req, res) => {
const dir = fs.opendirSync(fullPath)
let entity
let listing = []
while((entity = dir.readSync()) !== null) {
if(entity.isFile()) {
listing.push({ type: 'f', name: entity.name })
} else if(entity.isDirectory()) {
listing.push({ type: 'd', name: entity.name })
}
}
dir.closeSync()
// res.send(listing)
res.writeHead(200);
var template = `
<!doctype html>
<table border="1" margin: auto; text-align: center;>
<tr>
<th> Type </th>
<th> Name </th>
<th> Down </th>
<th> Del </th>
</tr>
`;
for(var i=1;i<listing.length;i++) {
template += `
<tr>
<th>${listing[i]['type']}</th>
<th>${listing[i]['name']}</th>
<th>
<form method='post' action='/downloadFile'>
<button type="submit" name='dlKey' value=${listing[i]['name']}>down</button>
</form>
</th>
<th>
<form method='post' action='/deleteFile'>
<button type="submit" name='dlKey' value=${listing[i]['name']}>del</button>
</form>
</th>
</tr>
`;
}
template +=`
</table>
`;
res.end(template);
})
app.post('/downloadFile', function(req,res) {
var filename = req.body.dlKey;
console.log(filename);
const directoryPath = fullPath + '/';
res.download(directoryPath + filename, filename, (err) => {
if (err) {
res.status(500).send({ ###파일이 존재하지 않다면
message: "Could not download the file. " + err,
});
}
});
})
app.post('/deleteFile', function(req,res) {
var filename = req.body.dlKey;
console.log(filename);
const directoryPath = fullPath + '/';
fs.unlink(directoryPath + filename, (err) => {
if (err) {
res.status(500).send({
message: "Could not delete the file. " + err,
});
}
// res.status(200).send({ ###존재하면
// message: "File is deleted.",
// });
res.redirect('/list'); ###인터넷에 바로 보여지게끔 List를 만들어줌
});
})
module.exports = app;
# rm -rf package.json
# npm install express morgan path body-parser cookie-parser multer ejs
# nodemon app.js
### npm publish
https://www.npmjs.com
# mkdir /allnew/node/impelfin-npm-deploy
# cd /allnew/node/impelfin-npm-deploy
# npm init -y
# npm install commander
# mkdir bin
## bin/cli.js
#!/usr/bin/env node
const { program } = require("commander");
program.action(cmd => console.log('✓ Running!!'));
program.parse(process.argv);
## package.json
{
"name": "impelfin-npm-deploy",
"version": “1.0.1”,
"description": "",
"main": "index.js",
"bin": {
"log-run": "bin/cli.js"
},
"keywords": [
"npm-deploy"
],
"author": "",
"license": "MIT",
"files" :[
"cli"
],
"dependencies": {
"commander": "^11.0.0"
}
}
# npm link
# log-run
✓ Running!!
# npm ls -g --depth=0
├── impelfin-npm-deploy@1.0.0 -> ./../../../../../../allnew/node/impelfin-npm-deploy
## .npmignore
.idea
node_modules
# npm login
npm notice Log in on https://registry.npmjs.org/
Login at:
https://www.npmjs.com/login?next=/login/cli/95eb7340-a0d4-4a9a-a5a9-e8f57a97afe8
Press ENTER to open in the browser...
Logged in onhttps://registry.npmjs.org/.
# npm whoami
impelfin
# npm publish (성공 메세지)
npm notice
npm notice 📦 impelfin-npm-deploy@1.0.1
npm notice === Tarball Contents ===
npm notice 146B bin/cli.js
npm notice 299B package.json
npm notice === Tarball Details ===
npm notice name: impelfin-npm-deploy
npm notice version: 1.0.1
npm notice filename: impelfin-npm-deploy-1.0.1.tgz
npm notice package size: 408 B
npm notice unpacked size: 445 B
npm notice shasum: d93b84a65f7dd40309db167a4df86b23967e671f
npm notice integrity: sha512-zQr+LU+GSbkfd[...]63XyQmNAsJCGg==
npm notice total files: 2
npm notice
npm notice Publishing to https://registry.npmjs.org/with tag latest and default access
+ impelfin-npm-deploy@1.0.1
## npmjs.com -> login -> Packages에서 확인
# cd ..
# mkdir impelfin-npm-test
# npm init -y
# npm install impelfin-npm-deploy@1.0.1
# cat package.json
{
"name": "impelfin-npm-test",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"impelfin-npm-deploy": "^1.0.1"
}
}
## 데이터명 : 기상청_일기도 조회서비스
token
endpoint(url) 잘확인하기
# cd /allnew/node
# cp -r mongoose weather
# rm -rf node_modules package* users_insert
# cd weather
## app.js
const express = require('express')
const morgan = require('morgan')
const path = require('path')
const app = express()
const bodyParser = require('body-parser')
const cookieParser = require('cookie-parser')
app.set('port', process.env.PORT || 8000)
app.use(morgan('dev'))
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: false }))
app.use(cookieParser())
app.use(express.static(path.join(__dirname, 'public')))
//mongoose configuration
const mongoose = require("mongoose")
mongoose.connect("mongodb://3.37.236.215:27017/mydb")
var main = require('./routes/main.js')
app.use('/', main)
app.listen(app.get('port'), () => {
console.log('8000 Port : Server Started...')
});
## public/index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>Weather Status</title>
</head>
<body>
<h1>Weather Status</h1>
<hr />
<form name="dataform">
<input type="button" value="Weather - API get data" onclick='document.targetURL.location.href="/getdata"'/>
<p>
<input type="button" value="Weather - MongoDB Collection Data" onclick='document.targetURL.location.href="/list"'/>
</p>
</form>
<hr />
<iframe name="targetURL" class="iframe-preview center" width="600" height="1100" style="border: none;" src="/list">
</iframe>
</body>
</html>
## routes/main.js
const express = require("express")
const app = express.Router()
const mongoose = require("mongoose")
const request = require('request')
const moment = require("moment")
const dateutil = require("data-utils")
const mongoClient = require('mongodb').MongoClient
let day = new Date().toLocaleDateString('sv').replaceAll('-',''); ### 긴텍스트들이 안에 이상하게 붙어있는거를 제고하기 위해 ReplaceAll 하기 위함
var keys = "B%2FNiJnYmkZV1%2FK7ulvZI4MoSXvCTDfNAd0Snw%2Bk6g4%2BbMk1LoGVhd75DJahjv4K35Cr9jh9RX0j%2BM89grKBYsw%3D%3D"; ##공공데이터포털은 API 노출되도 상관없음
var url = " https://apis.data.go.kr/1360000/WthrChartInfoService/getSurfaceChart ";
var queryParams ='?' + encodeURIComponent('serviceKey') + "=" + keys;
queryParams +='&' + encodeURIComponent('pageNo') + "=" + encodeURIComponent('1');
queryParams +='&' + encodeURIComponent('numOfRows') + "=" + encodeURIComponent('10');
queryParams +='&' + encodeURIComponent('dataType') + "=" + encodeURIComponent('JSON');
queryParams +='&' + encodeURIComponent('code') + "=" + encodeURIComponent('3');
queryParams +='&' + encodeURIComponent('time') + "=" + encodeURIComponent(day);
// define schema
var DataSchema = mongoose.Schema({
day_v : String,
imgSrc1_v : String,
imgSrc2_v : String
})
var Data = mongoose.model('weathers', DataSchema); ##MongoDB는 복수형으로만들기, weather(X) weathers(O)
// getdata
app.get('/getdata', function (req, res, next) {
request({
url : url + queryParams,
method : "GET"
}, function (error, response, body) {
Data.find({}).remove().exec();
if (error) console.log(error);
// console.log(body);
let data = JSON.parse(body);
let imgSrcArr = data['response']['body']['items']['item'][0]['man-file'].split(','); ###body를 잘 찍기 (split쪼갠거에 1st
let imgSrc1 = imgSrcArr[0].slice(1);
let imgSrc2 = imgSrcArr[1].slice(1);
console.log("imgSrc1 : " + imgSrc1);
console.log("imgSrc2 : " + imgSrc2);
res.writeHead(200);
var template =`
<!doctype html>
<html>
<head>
<title>Result</title>
<meta charset="urf-8">
</head>
<body>
<img src="${imgSrc1}" width="500" height="500"></img><p>
<img src="${imgSrc2}" width="500" height="500"></img><p>
</body>
</html>
`;
res.end(template);
var newData = new Data({day_v : day, imgSrc1_v : imgSrc1, imgSrc2_v : imgSrc2})
newData.save(function(err, result) {
if (err) return console.error(err)
console.log(new Date(), result)
})
})
})
// list
app.get('/list', function(req, res, next) {
Data.findOne({}, function(err, docs) {
if(err) console.log('err');
console.log(docs)
res.writeHead(200);
var template =`
<!doctype html>
<html>
<head>
<title>Result</title>
<meta charset="urf-8">
</head>
<body>
<img src="${docs['imgSrc1_v']}" width="500" height="500"></img><p>
<img src="${docs['imgSrc2_v']}" width="500" height="500"></img><p>
</body>
</html>
`;
res.end(template);
})
})
module.exports = app;
# npm init -y
# npm install express morgan path body-parser cookie-parser request mongoose moment data-utils mongodb
# nodemon app.js
# cd ..
# git add .
# git commit -m "weather"
# git push
## master…
# cd /allnew/node/
# git fetch
# git pull
# tar cvzf weather.tar.gz weather
# cd /allnew/docker
# mkdir k8s-weather
# cd k8s-weather/
# mv /allnew/node/weather.tar.gz .
# cp ../k8s-remote/* .
# rm -rf remoteapi.tar.gz
## Dockerfile
1 # step 1 : Base Images
2 FROM node:18.16.0
3
4 # step 2 : Package Install
5 RUN apt -y update && apt -y upgrade && apt -y install git net-tools vim
6
7 # step 3 : Specify a working directory
8 WORKDIR '/root'
9
10 # step 4 : Config file copy
11 COPY weather.tar.gz .
12
13 # step 5 : install express
14 RUN tar xvzf weather.tar.gz
15 WORKDIR '/root/weather'
16 RUN npm install
17 RUN npm install -g nodemon
18
19 # Step 6 : open port
20 EXPOSE 8000
21
22 # Step 7 : Execution Program
23 CMD ["nodemon", "app.js"]
## Makefile
1 build:
2 docker build -t weather .
3 run:
4 docker run -it -d -p 8000:8000 --name weather weather
5 exec:
6 docker exec -it weather /bin/bash
7 logs:
8 docker logs weather
9 ps:
10 docker ps -a
11 img:
12 docker images
13 rm:
14 docker rm -f $$(docker ps -aq)
15 rmi:
16 docker rmi $$(docker images -q)
# make
# make img
# make run
# make ps
# docker commit weather weathers
# docker tag weathers impelfin/weathers
# make img
# docker push impelfin/weathers
# make rm
# make img
# docker rmi impelfin/weathers
# make rmi
# make img
## deployment.yaml
1 apiVersion: apps/v1
2 kind: Deployment
3 metadata:
4 name: weathers
5 labels:
6 app: weathers
7 spec:
8 replicas: 3
9 selector:
10 matchLabels:
11 app: weathers
12 template:
13 metadata:
14 labels:
15 app: weathers
16 spec:
17 containers:
18 - name: weathers
19 image: impelfin/weathers
20 ports:
21 - containerPort: 8000 # Contianer Port(pod port)
## service.yaml
1 apiVersion: v1
2 kind: Service
3 metadata:
4 name: weathers
5 labels:
6 run: weathers
7 spec:
8 type: NodePort # service type
9 ports:
10 - nodePort: 31200 # outter port
11 port: 8080 # service port
12 targetPort: 8000 # container port (pod port)
13 protocol: TCP
14 selector:
15 app: weathers
16 type: LoadBalancer
17 externalIPs:
18 - 15.165.183.87
# kubectl apply -f deployment.yaml
# kubectl apply -f service.yaml
# kubectl get deployment
# kubectl get svc -o wide
# kubectl get pod -o wide