From 3431e667a5c6475043ebfd97b43a3fdc4b078596 Mon Sep 17 00:00:00 2001 From: Phil Burton Date: Mon, 25 Feb 2019 13:37:59 +0000 Subject: Refactor and clean up notes --- day1/diving-deep-into-blockchain.txt | 100 +++++++++++++++++++++ day1/first-talk.txt | 105 ----------------------- day1/massively-scaled-microservices.txt | 108 +++++++++++++++++++++++ day1/no-sql.txt | 57 ------------ day1/non-scalar-data.txt | 59 +++++++++++++ day1/second-talk.txt | 97 --------------------- day1/supervised-learning.txt | 77 +++++++++++++++++ day2/building-first-class-rest-apis.txt | 143 +++++++++++++++++++++++++++++++ day2/first-talk.txt | 42 --------- day2/from-dev-to-prod-with-gitlab-ci.txt | 124 +++++++++++++++++++++++++++ day2/gitlab.txt | 120 -------------------------- day2/microservices-gone-wrong.txt | 43 ++++++++++ day2/more-secrets-of-crpyto.txt | 53 ++++++++++++ day2/rest-apis.txt | 142 ------------------------------ day2/second-crypto.txt | 51 ----------- links-and-useful-keywords.txt | 30 +++++++ 16 files changed, 737 insertions(+), 614 deletions(-) create mode 100644 day1/diving-deep-into-blockchain.txt delete mode 100644 day1/first-talk.txt create mode 100644 day1/massively-scaled-microservices.txt delete mode 100644 day1/no-sql.txt create mode 100644 day1/non-scalar-data.txt delete mode 100644 day1/second-talk.txt create mode 100644 day1/supervised-learning.txt create mode 100644 day2/building-first-class-rest-apis.txt delete mode 100644 day2/first-talk.txt create mode 100644 day2/from-dev-to-prod-with-gitlab-ci.txt delete mode 100644 day2/gitlab.txt create mode 100644 day2/microservices-gone-wrong.txt create mode 100644 day2/more-secrets-of-crpyto.txt delete mode 100644 day2/rest-apis.txt delete mode 100644 day2/second-crypto.txt create mode 100644 links-and-useful-keywords.txt diff --git a/day1/diving-deep-into-blockchain.txt b/day1/diving-deep-into-blockchain.txt new file mode 100644 index 0000000..8fe9218 --- /dev/null +++ b/day1/diving-deep-into-blockchain.txt @@ -0,0 +1,100 @@ +Block chain + +Tomasz Kowalczyk +https://joind.in/event/php-uk-conference-2019/dive-deep-into-blockchain + +- Block chain is a database +- Bespoke data structure + +"Not crpyto" + +General Purpose block chain: + +Why? +- Data in block is immutable +- Don't edit or remove, you append +- traceability (Auditing / logging) +- Verifiability (Data must be fully valid) +- reproducibility +- decentralisation +- Trust-less (Removes need for trust) +- transparency + +How? +- What type? + - public - will be hacked + - shared - shared ledger + - private +- Cryptography + - pub/rivate keys + - sign information + - verifablity + - Elliptic Curves + - openssl ecparam -list_curves + - Addressing + - can shorten public keys massively + - Hash algorithm + - Integrity + - 2x sha256 (sha256(sha256($data))) + - 4B version, 32B hash of last Block, 32B hash of data block, 4B timestamp, 4B difficulty, 4B nonce + - Merkle tree - binary hash tree + - Merkle path - list of hashes through the tree + - Genesis block - First block in the chain + - Special as it's first + - Prevents exploitation by being hardcoded + +- consensus + - Confidence + - chain forking needs resolution + - bitcoin users longest chain wins + - proof of work / proof of stake + - difficulty is no of zeros that need to be at the start of the hash + - others (proof of burn, proof of time, proof of ....) + + +- Smart contracts + - Programs that are included in the chain + - e.g. A loan paid by the chain + +- Lighting network (drawback off blockhain is it focuses on integrity not perf) + - Off chain transactions with on-chain settlement + +- Not all blockchains are chains + - IOTA Tangle + +- Challenges + - Fault tolerance + - You lose control of public/shared blockchains + - Now way to enforce software updates + - Conflict resolution + - Integration with current systems + - Security + - Cryptography is first and last line of defence + +- Not perfect for: + - scalability + - performance + - throughput + +- Incentives + - Mining in currencies + - Power for money +- Privacy + - Protect data + +- Hard forks +- 51% attacks + +- Right solution for the right kind of problem + +- Event sourcing on steroids + +QUESTIONS +- Any php recommendations? + - No. +- security? + - Store data not values somewhere else + - Encrypt +- Why private blocks? + - Can give public list of hashes for verification + - keep data private diff --git a/day1/first-talk.txt b/day1/first-talk.txt deleted file mode 100644 index da8689e..0000000 --- a/day1/first-talk.txt +++ /dev/null @@ -1,105 +0,0 @@ -# micro services - perf - -## Overivew - -glu mobile - microservices - -- Use docker -- HTTP processing -- Amazon ECS -- NOSQL Redis + couch base -- 40M votes a day -- 1M active users -- 14K req/m - -## Microservices - -- Small -- Simole -- easy to deploy -- upgradable - -Use Base images -Name by version and never change (Freeze them) - -## background processing -- Execute externally -- Shutdown functions -- Job server -- fastcgi_finish_request - -## HTTP Compression - -- gzip on -- gzip_types -- gzip_proxied any -- gzip_comp_level 5 -- gzip_min_length (default 20) -- Smaller than 1500B means it will always fit in one TCP packet - -If content_length is not set then nginx will compress always - -Laravel/Lume/Others add middleware to add content_length - -## Caching -- HTTP2 Server push -- Repeated server calls HTTP1 -- 304s - - Last-modified - - cache-control - -## No SQL -- Mongo/Redis -- Fight network I/O -- save hdd space -- compress large messages -- serialise better -- message templates -- add ttl expiry -- shorten cache keys - -## Fast caching -- PHP Redis -- Predis - high memory, slower -- Persistent connections available - -## Deployment EC8 -- 0 Downtime -- quick rollout -- ELB health-check -- Auto scaling - -## Hardware upgrades -- Doubling the hardware, halving the # of machines - -## VPC -- Closer together -- Same location -- utilise internal networks -- other thoughts - - Async I/O - - swoole - - HTTP/2 - - Tars - - binary (80% reduction) - -## Managing images -- Don't use aptitude (apt) -- Build from source -- fpm and Nginx together - - Separate not helpful - - No need for TCP/IP overhead -- supervisorD - everywhere - -# Useful terms -aerospike -couch base -php_swoole -supervisorD -new relic -bugsnag -amazon cloud watch -sonarqube -sensolabs security checker -fastcgi_finish_request -crowdstar https://github.com/Crowdstar/background-processing diff --git a/day1/massively-scaled-microservices.txt b/day1/massively-scaled-microservices.txt new file mode 100644 index 0000000..3b6dbe0 --- /dev/null +++ b/day1/massively-scaled-microservices.txt @@ -0,0 +1,108 @@ +# micro services - perf + +https://joind.in/event/php-uk-conference-2019/massively-scaled-high-performance-web-services-with-php +https://www.slideshare.net/DeminYin/massively-scaled-high-performance-web-services-with-php-132696547 + +## Overivew + +glu mobile - microservices + +- Use docker +- HTTP processing +- Amazon ECS +- NOSQL Redis + couch base +- 40M votes a day +- 1M active users +- 14K req/m + +## Microservices + +- Small +- Simole +- easy to deploy +- upgradable + +Use Base images +Name by version and never change (Freeze them) + +## background processing +- Execute externally +- Shutdown functions +- Job server +- fastcgi_finish_request + +## HTTP Compression + +- gzip on +- gzip_types +- gzip_proxied any +- gzip_comp_level 5 +- gzip_min_length (default 20) +- Smaller than 1500B means it will always fit in one TCP packet + +If content_length is not set then nginx will compress always + +Laravel/Lume/Others add middleware to add content_length + +## Caching +- HTTP2 Server push +- Repeated server calls HTTP1 +- 304s + - Last-modified + - cache-control + +## No SQL +- Mongo/Redis +- Fight network I/O +- save hdd space +- compress large messages +- serialise better +- message templates +- add ttl expiry +- shorten cache keys + +## Fast caching +- PHP Redis +- Predis - high memory, slower +- Persistent connections available + +## Deployment EC8 +- 0 Downtime +- quick rollout +- ELB health-check +- Auto scaling + +## Hardware upgrades +- Doubling the hardware, halving the # of machines + +## VPC +- Closer together +- Same location +- utilise internal networks +- other thoughts + - Async I/O + - swoole + - HTTP/2 + - Tars + - binary (80% reduction) + +## Managing images +- Don't use aptitude (apt) +- Build from source +- fpm and Nginx together + - Separate not helpful + - No need for TCP/IP overhead +- supervisorD - everywhere + +# Useful terms +aerospike +couch base +php_swoole +supervisorD +new relic +bugsnag +amazon cloud watch +sonarqube +sensolabs security checker +fastcgi_finish_request +crowdstar https://github.com/Crowdstar/background-processing diff --git a/day1/no-sql.txt b/day1/no-sql.txt deleted file mode 100644 index ccb6a4a..0000000 --- a/day1/no-sql.txt +++ /dev/null @@ -1,57 +0,0 @@ -# Non-Scalar Data - -## Redis sets SSAD - -- key binding - safe string -- sets of data -- taggable - -## Document data stores -- Usually JSON -- Richer -- MongoDB - single master -- CouchDB - multi master -- ES - text search index - -## MongoDB -- pecl/mongodb -- mongodb/mongodb - -docs are PHP objects or assoc arrays - -## Relational Database -- normalisation - -- postgres - - HSTORE - - JSON/JSONB - - -# Querying data - -Sets - SISMEMBER and SMEMBER -- Aggregations -- buckets -- postgres - non standard - -## Redis -- Can add to sets -- Atomic -- fast -- counters - -DONT: RETRIVE MANIPULATE STORE - - Do we do this at d3R? - - redis bad at scaling - - joned.in/25887 - - -# Useful links -https://redis.io/commands/sadd -https://www.postgresql.org/docs/9.1/hstore.html -https://wiki.openstreetmap.org/wiki/PostgreSQL -https://en.m.wikipedia.org/wiki/Benford%27s_law -https://php-ml.readthedocs.io/en/latest/machine-learning/workflow/pipeline/ diff --git a/day1/non-scalar-data.txt b/day1/non-scalar-data.txt new file mode 100644 index 0000000..908cf84 --- /dev/null +++ b/day1/non-scalar-data.txt @@ -0,0 +1,59 @@ +# Non-Scalar Data +https://joind.in/event/php-uk-conference-2019/storing-non-scalar-data +Derick Rethans + +## Redis sets SSAD + +- key binding - safe string +- sets of data +- taggable + +## Document data stores +- Usually JSON +- Richer +- MongoDB - single master +- CouchDB - multi master +- ES - text search index + +## MongoDB +- pecl/mongodb +- mongodb/mongodb + +docs are PHP objects or assoc arrays + +## Relational Database +- normalisation + +- postgres + - HSTORE + - JSON/JSONB + + +# Querying data + +Sets - SISMEMBER and SMEMBER +- Aggregations +- buckets +- postgres - non standard + +## Redis +- Can add to sets +- Atomic +- fast +- counters + +DONT: RETRIVE MANIPULATE STORE + + Do we do this at d3R? + + redis bad at scaling + + joned.in/25887 + + +# Useful links +https://redis.io/commands/sadd +https://www.postgresql.org/docs/9.1/hstore.html +https://wiki.openstreetmap.org/wiki/PostgreSQL +https://en.m.wikipedia.org/wiki/Benford%27s_law +https://php-ml.readthedocs.io/en/latest/machine-learning/workflow/pipeline/ diff --git a/day1/second-talk.txt b/day1/second-talk.txt deleted file mode 100644 index 46df43f..0000000 --- a/day1/second-talk.txt +++ /dev/null @@ -1,97 +0,0 @@ -Block chain - -- Block chain is a database -- Bespoke data structure - -"Not crpyto" - -General Purpose block chain: - -Why? -- Data in block is immutable -- Don't edit or remove, you append -- traceability (Auditing / logging) -- Verifiability (Data must be fully valid) -- reproducibility -- decentralisation -- Trust-less (Removes need for trust) -- transparency - -How? -- What type? - - public - will be hacked - - shared - shared ledger - - private -- Cryptography - - pub/rivate keys - - sign information - - verifablity - - Elliptic Curves - - openssl ecparam -list_curves - - Addressing - - can shorten public keys massively - - Hash algorithm - - Integrity - - 2x sha256 (sha256(sha256($data))) - - 4B version, 32B hash of last Block, 32B hash of data block, 4B timestamp, 4B difficulty, 4B nonce - - Merkle tree - binary hash tree - - Merkle path - list of hashes through the tree - - Genesis block - First block in the chain - - Special as it's first - - Prevents exploitation by being hardcoded - -- consensus - - Confidence - - chain forking needs resolution - - bitcoin users longest chain wins - - proof of work / proof of stake - - difficulty is no of zeros that need to be at the start of the hash - - others (proof of burn, proof of time, proof of ....) - - -- Smart contracts - - Programs that are included in the chain - - e.g. A loan paid by the chain - -- Lighting network (drawback off blockhain is it focuses on integrity not perf) - - Off chain transactions with on-chain settlement - -- Not all blockchains are chains - - IOTA Tangle - -- Challenges - - Fault tolerance - - You lose control of public/shared blockchains - - Now way to enforce software updates - - Conflict resolution - - Integration with current systems - - Security - - Cryptography is first and last line of defence - -- Not perfect for: - - scalability - - performance - - throughput - -- Incentives - - Mining in currencies - - Power for money -- Privacy - - Protect data - -- Hard forks -- 51% attacks - -- Right solution for the right kind of problem - -- Event sourcing on steroids - -QUESTIONS -- Any php recommendations? - - No. -- security? - - Store data not values somewhere else - - Encrypt -- Why private blocks? - - Can give public list of hashes for verification - - keep data private diff --git a/day1/supervised-learning.txt b/day1/supervised-learning.txt new file mode 100644 index 0000000..1dce7da --- /dev/null +++ b/day1/supervised-learning.txt @@ -0,0 +1,77 @@ +# Learning: the hows and whys of machine learning + +Liam Wiltshire +https://liam-wiltshire.github.io/talks/?talk=machinelearning&conference=phpuk +https://joind.in/event/php-uk-conference-2019/learning-the-hows-and-whys-of-machine-learning + +## Overivew + +Charge backs + +## Supervised learning +Training data +Learning functions +Categorisation / Classification +Regression - Where do we sit on a line + +## Naive Bayes classifier +Standardise words +- Un pluralise +- Un gender +- Un tense +- etc + +More data == better + +## Tokenisation +https://en.wikipedia.org/wiki/Benford%27s_law +https://php-ml.readthedocs.io + +Unique tokens for each unique context + +## Imbalanced data +One category has more database +99% data not charge back +Just being accurate, not very helpful + - Started by flagging 100% as fine. + - Need to collect more data, change methods, resample data + +## Understand data +- context +- Common data vs specific data +- Continuous vs discrete data + +## KNN +K Nearest Number +https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm + - Distances + - less sensitive to imbalance + - Keep K odd (no draws) + +## Handling nominal data + +Binary +- Increase amounts of dimensions +- normalisation required +- equal scales + +## Contextless data is meaningless +Is it normal? + +## Next to try +Weighting +Different dimensions +Change K value (was 3NN) +Remove outliers +Diff distance function +weighted distance + + + + +# Useful links +https://en.wikipedia.org/wiki/Benford%27s_law +https://php-ml.readthedocs.io +https://liam-wiltshire.github.io/talks/?talk=machinelearning&conference=phpuk +https://joind.in/event/php-uk-conference-2019/learning-the-hows-and-whys-of-machine-learning +https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm diff --git a/day2/building-first-class-rest-apis.txt b/day2/building-first-class-rest-apis.txt new file mode 100644 index 0000000..681b818 --- /dev/null +++ b/day2/building-first-class-rest-apis.txt @@ -0,0 +1,143 @@ +# First class rest APIs with symfony + +## Who +- @michaelcullumuk +- Michael Cullum +- Works for bud. (2 weeks ago) +- core team at syfony +- fig working group +- fig security + +## WTF IS REST? + +- Paper by roy fielding +- dry and boring +- Representational State Transfer (REST) +- Communicating state + +### How to comply to REST + +- Uniform interface + - Each api works the same at a ui level + - Gives all data to client that client can then use to give back and modify + - use mime types + - Hyperdata. Link to other data. + +- Stateless + - No session + - Self contained in one request + - Auth in the request + +- Cacheable + - If not cacheable, you need to tell clients this + +- Client-Server need to be distinguishable + - Independently build client and server + +- Layered system + - May not be talking to an end user + - May be other applications or proxies or API in between + - (like micro services talk adding auth or similar at certain APIs) + +- Code on demand + - Be able to run code on the server + - Legacy. + - bit weird + - considered optional + +- USE HTTP WELL + - HTTP is your friend + - RESTful does not discuss http + - HTTP VERBS + - USE THEM ALL + - (Google crawling with GET, where they hit a GET verm delete end point) + - USE RESPONSE CODES + - Rate Limiting + use codes (429) + - 503 for when there's issues + +- In symfony + - Use the constants + +## Errors +Exception controller +Does logging +Return our json response +getErrors() handles error once in one function, included for all controllers + +## DTO ALL THE THINGS +- in PHP we tend to use arrays lots +- Decode some JSON and might put it into array +- bad as we have no typing +- We can use typing to move complication and validation of data to PHP level +- Reduces typos etc +- In symfony we can auto-hydrate stuff + - mapAndPersist + - A dataTransferObjectConvertor + +## Validation +- Exceptions + - Exceptional case + - Should expect edge cases and failings + - Remove control from controllers + - isValid in the controller forces us to have logic in the controller + - instead try and persist an object, throw new exception + +## Output + - Symfony serialiser component + - Choose output types easily + - Transformers to migrate the data from an entity and map to response data + - Allows us to focus on API respires not database layer entities + - PHPLeaue FRACTAL (for transformers) + - Transformers allow us to be explicit! + - Using fractal can build our json but bhandke collections, pagination etc + +## Nobody likes pagination + - Pain in the arse + - Clients need to work out logic for reading + - Server side we have extra logic, limits etc + + How to do it then? + - Do it + - Link to next and previous (hyper media) + - Detail current page + - Detail total number of pages + - Assuming you know + - Disabling totals speeds things up + - Perhaps client option + - Use paginator tools + - Really simple controllers + +## Sorts and filters +- query string + - pagiantion + - sorting + - includes + - filters + +## Tools +- Api platform + - EZ PZ +- FOS REST + - automatic routing + - slugs etc + +# RECAP +PRINCIPLES OF REST +USE HTTP WELL - verbs and codes +ERROR handling, abstracting validation. Handle expectations for response codes +DTOs and param convertor +Validation bubbles, catch and re-throw +Formatting output with transforms +Pagination - annoying but easy, if we p[ass via repos. +Sorts and filters similar to Pagination +Tools!!! Use em.] + + + +# Useful Link +https://github.com/FriendsOfSymfony/FOSRestBundle +https://github.com/whiteoctober/Pagerfanta +https://www.ics.uci.edu/~fielding/pubs/dissertation/rest_arch_style.htm +https://en.wikipedia.org/wiki/Data_transfer_object +https://fractal.thephpleague.com/ +https://api-platform.com/ diff --git a/day2/first-talk.txt b/day2/first-talk.txt deleted file mode 100644 index 1cb53b3..0000000 --- a/day2/first-talk.txt +++ /dev/null @@ -1,42 +0,0 @@ -Microservices gone wrong - -- starting from scratch - -api gateway -middleware for extrenal requests -message-bus rabbitMQ - -Service oer domain entity - -Meta service using ES -Kurbenetes - -Why local dev failed? -Slow. -Incosistent -Would break - -mainly: "Someone elses problem" - all devs - - -Production release took 1 week -Cost of context switching - -Smoke Test - -- Lessons Learned - - - service call's are unreliable - - "microlyth" - -1. DOnt do micorservices - a. unless you have a dedicated tooling and automation team -2. Start with big services - a/ Split if you require -3. Auomate everything - a spin up, deployment migration, backup state resortation Elliptic -4. Dont plan for failure, live it - a. failure modes should bebuilt first, tested first, and relied upon -5. Define SLO early - a. Define business objectives for each service and system before building - diff --git a/day2/from-dev-to-prod-with-gitlab-ci.txt b/day2/from-dev-to-prod-with-gitlab-ci.txt new file mode 100644 index 0000000..f7e25ba --- /dev/null +++ b/day2/from-dev-to-prod-with-gitlab-ci.txt @@ -0,0 +1,124 @@ +# Dev to prod with GitLab CI + +https://talks.bitexpert.de/phpuk19-gitlabci/ +Stephan Hockdorfer + +## Overview +- Stefan +- bitExpert AG (Germany) + +## Gitlab +Old wikipedia description: + Web based git repo manager with wiki and issue tracking +## Beyond CI/CD + - Dev, Ops, CD/CO + +## Installation +Lots of ways. +Docker. +- Use a registry +- Sonatype Nexus +- Nexus installation for repo manager +- Reverse proxy for routing, and running multiple instances + +### Traefik +Simple to configure +Configure +Can listen to socket, and other things +Can do lets encrypt stuff + + +## Crating projects +Magento as an example +Slightly more difficult to install +Has some complication + +1. Composer install +2. Add to git and push +3. Need a git lab runner (Can host the runners or use SASS offering) + (docker container for runner) + +(Gitlab instance and runner installed via docker) + +Access runner overview page and grab registration token +Create runner instance +gitlab-runner gitlab-runner register + - token + - description + - tags + - run untagged builds? + - current project? + - executor? (docker, shell,.. etc) (more control if we use docker) + - default docker image +-- This creates the runner config file +-- Edit and amend, e.g. add volumes of shared composer stuff ***Should we do this at d3r?*** + +## Managing Secrets +add to project, can use in groups + +## Add .gitlab-ci.yaml (similar to travis) + -- define Image + -- define job scripts + +## http-basic auth via composer +What is this? + + +## Gitlab services +Multiple images should spin up first, then jobs can run +Health checks built in, cheks exposed ports +Spins up second container waiting for ports to be available then spins down +(Gitlab says you cant 100% trust this to coinfirm all is working) + +### add mysql service +1. Define image, define variables (passowrd) +2. define things it needs + +## Build piplines +- test +- build +- build staging + - define where in "data center" the staging stuff is + - Define git registration + - dine staging url +- Build creates tar file ready for deploy +- deploy! +- when manual trigger - Dont auto build and deploy everything to staging and production, Allow us to test te staging +env first and then manually kick off the deployment for production + + +## Problem with this set-up +- Multi developers means we're not sure of staging set-up +- Staging per developer? +- What about a staging per feature branch? + - yes + - Review Apps + - documentation not great + - Dynamically create staging environments + - make sure on_stop + - only run for merge_requests (fairly new, used to have to do "all branches but not master") + - define stop_review to pull down containers + +1. Create merge request +2. Test and build begins +3. It then gives you the urls for testing +4. Merge. Then staging is deployed +5. Then manually hit up production + + + +## Questions +Create a standard web app +Can Import from github!!! +Auto dev ops? + - Auto turned on + - Not sure how good it is + - maybe we just disable +Can we lock down how things get deployed + - Role based auth is bad + - Don't have fine grain control (most of the team need maintainer role) + +# Useful links +https://www.sonatype.com/ +https://traefik.io/ +https://talks.bitexpert.de/phpuk19-gitlabci/ diff --git a/day2/gitlab.txt b/day2/gitlab.txt deleted file mode 100644 index 2ce3906..0000000 --- a/day2/gitlab.txt +++ /dev/null @@ -1,120 +0,0 @@ -# Dev to prod with GitLab CI - -## Overview -- Stefan -- bitExpert AG (Germany) - -## Gitlab -Old wikipedia description: - Web based git repo manager with wiki and issue tracking -## Beyond CI/CD - - Dev, Ops, CD/CO - -## Installation -Lots of ways. -Docker. -- Use a registry -- Sonatype Nexus -- Nexus installation for repo manager -- Reverse proxy for routing, and running multiple instances - -### Traefik -Simple to configure -Configure -Can listen to socket, and other things -Can do lets encrypt stuff - - -## Crating projects -Magento as an example -Slightly more difficult to install -Has some complication - -1. Composer install -2. Add to git and push -3. Need a git lab runner (Can host the runners or use SASS offering) - (docker container for runner) - -(Gitlab instance and runner installed via docker) - -Access runner overview page and grab registration token -Create runner instance -gitlab-runner gitlab-runner register - - token - - description - - tags - - run untagged builds? - - current project? - - executor? (docker, shell,.. etc) (more control if we use docker) - - default docker image --- This creates the runner config file --- Edit and amend, e.g. add volumes of shared composer stuff ***Should we do this at d3r?*** - -## Managing Secrets -add to project, can use in groups - -## Add .gitlab-ci.yaml (similar to travis) - -- define Image - -- define job scripts - -## http-basic auth via composer -What is this? - - -## Gitlab services -Multiple images should spin up first, then jobs can run -Health checks built in, cheks exposed ports -Spins up second container waiting for ports to be available then spins down -(Gitlab says you cant 100% trust this to coinfirm all is working) - -### add mysql service -1. Define image, define variables (passowrd) -2. define things it needs - -## Build piplines -- test -- build -- build staging - - define where in "data center" the staging stuff is - - Define git registration - - dine staging url -- Build creates tar file ready for deploy -- deploy! -- when manual trigger - Dont auto build and deploy everything to staging and production, Allow us to test te staging -env first and then manually kick off the deployment for production - - -## Problem with this set-up -- Multi developers means we're not sure of staging set-up -- Staging per developer? -- What about a staging per feature branch? - - yes - - Review Apps - - documentation not great - - Dynamically create staging environments - - make sure on_stop - - only run for merge_requests (fairly new, used to have to do "all branches but not master") - - define stop_review to pull down containers - -1. Create merge request -2. Test and build begins -3. It then gives you the urls for testing -4. Merge. Then staging is deployed -5. Then manually hit up production - - - -## Questions -Create a stadard web app -Can Import from github!!! -Autoi dev ops? - - Auto turned on - - Not sure how good it is - - maybe we just disable -Can we lock down how things get deployed - - ROle based auth is bad - - Dont have fine grain control (most of the team need nainainer role) - -# Useful links -https://www.sonatype.com/ -https://traefik.io/ diff --git a/day2/microservices-gone-wrong.txt b/day2/microservices-gone-wrong.txt new file mode 100644 index 0000000..88a5096 --- /dev/null +++ b/day2/microservices-gone-wrong.txt @@ -0,0 +1,43 @@ +# Microservices gone wrong + +Anthony Ferrara +https://docs.google.com/presentation/d/1Ogejf47b7k0RWU-7lE_uBsCmJxL3CvydOl8fVukuQ_4/edit + +- starting from scratch + +api gateway +middleware for external requests +message-bus rabbitMQ + +Service per domain entity + +Meta service using ES +Kurbenetes + +Why local dev failed? +Slow. +Inconsistent +Would break + +mainly: "Someone elses problem" - all devs + +Production release took 1 week +Cost of context switching + +Smoke Test + +- Lessons Learned + + - service call's are unreliable + - "microlyth" + +1. Don't do micorservices + a. unless you have a dedicated tooling and automation team +2. Start with big services + a/ Split if you require +3. Automate everything + a spin up, deployment migration, backup state restoration Elliptic +4. Don't plan for failure, live it + a. failure modes should rebuilt first, tested first, and relied upon +5. Define SLO early + a. Define business objectives for each service and system before building diff --git a/day2/more-secrets-of-crpyto.txt b/day2/more-secrets-of-crpyto.txt new file mode 100644 index 0000000..0246a58 --- /dev/null +++ b/day2/more-secrets-of-crpyto.txt @@ -0,0 +1,53 @@ +# Crypto +Christopher Riley +@giveupalready +https://github.com/carnage + +## Overview: +Why do we need crypto? + +- Confidentiality +- Key Ex +- Identity +- Authentication + +## Asymmetric Crypto +- Symmetric crypto faster, asymmetric slower and computational expensive +- RSA + - slightly old + - Needs 2048+ size keys + +## Elliptic curve crypto +- Dot function +- Key exchange with shared key created from a -> aG -> aGb <- bG <- b +- Week to man in the middle + +## Web of trust +- How can we safely pass the key over an unsecured channel +- Just blindly trust on first use (like ssh) + +## The trusted third party +- Get a 3rd actor to meet Alice and Bob to add trust + +## hash functions +- md5 weak +- sha good + +# Merkel tree +- Hash chunks +- Combine chunks and hash +- Can work out which parts are wrong +- good for static data +- hash tree + +## Block chain +- Use block chain to share trusted messages + + + + + + + +-------- +Web of trust, trust once, trust forever diff --git a/day2/rest-apis.txt b/day2/rest-apis.txt deleted file mode 100644 index a087a52..0000000 --- a/day2/rest-apis.txt +++ /dev/null @@ -1,142 +0,0 @@ -# First class rest APIs with symfony - -## Who -- @michaelcullumuk -- Works for bud. (2 weeks ago) -- core team at syfony -- fig working group -- fig security - -## WTF IS REST? - -- Paper by roy fielding -- dry and boring -- Representational State Transfer (REST) -- Communicating state - -### How to comply to REST - -- Uniform interface - - Each api works the same at a ui level - - Gives all data to client that client can then use to give back and modify - - use mime types - - Hyperdata. Link to other data. - -- Stateless - - No session - - Self contained in one request - - Auth in the request - -- Cacheable - - If not cacheable, you need to tell clients this - -- Client-Server need to be distinguishable - - Independently build client and server - -- Layered system - - May not be talking to an end user - - May be other applications or proxies or API in between - - (like micro services talk adding auth or similar at certain APIs) - -- Code on demand - - Be able to run code on the server - - Legacy. - - bit weird - - considered optional - -- USE HTTP WELL - - HTTP is your friend - - RESTful does not discuss http - - HTTP VERBS - - USE THEM ALL - - (Google crawling with GET, where they hit a GET verm delete end point) - - USE RESPONSE CODES - - Rate Limiting + use codes (429) - - 503 for when there's issues - -- In symfony - - Use the constants - -## Errors -Exception controller -Does logging -Return our json response -getErrors() handles error once in one function, included for all controllers - -## DTO ALL THE THINGS -- in PHP we tend to use arrays lots -- Decode some JSON and might put it into array -- bad as we have no typing -- We can use typing to move complication and validation of data to PHP level -- Reduces typos etc -- In symfony we can auto-hydrate stuff - - mapAndPersist - - A dataTransferObjectConvertor - -## Validation -- Exceptions - - Exceptional case - - Should expect edge cases and failings - - Remove control from controllers - - isValid in the controller forces us to have logic in the controller - - instead try and persist an object, throw new exception - -## Output - - Symfony serializer component - - Choose output types easily - - Transformers to migrate the data from an entity and map to response data - - Allows us to focus on API respires not database layer entities - - PHPLeaue FRACTAL (for transformers) - - Transformers allow us to be explicit! - - Using fractal can build our json but bhandke collections, pagination etc - -## Nobody likes pagination - - Pain in the arse - - Clients need to work out logic for reading - - Server side we have extra logic, limits etc - - How to do it then? - - Do it - - Link to next and previous (hyper media) - - Detail current page - - Detail total number of pages - - Assuming you know - - Disabling totals speeds things up - - Perhaps client option - - Use paginator tools - - Really simple controllers - -## Sorts and filters -- query string - - pagiantion - - sorting - - includes - - filters - -## Tools -- Api platform - - EZ PZ -- FOS REST - - automatic routing - - slugs etc - -# RECAP -PRINCIPLES OF REST -USE HTTP WELL - verbs and codes -ERROR handling, abstracting validation. Handle expections for response codes -DTOs and param convertor -Validation bubbles, catch and re-throw -Fromatting output wiht transformes -Pagination - annoying but easy, if we p[ass via repos. -Sorts and filters similar to Pagination -Tools!!! Use em.] - - - -# Useful Link -https://github.com/FriendsOfSymfony/FOSRestBundle -https://github.com/whiteoctober/Pagerfanta -https://www.ics.uci.edu/~fielding/pubs/dissertation/rest_arch_style.htm -https://en.wikipedia.org/wiki/Data_transfer_object -https://fractal.thephpleague.com/ -https://api-platform.com/ diff --git a/day2/second-crypto.txt b/day2/second-crypto.txt deleted file mode 100644 index 18d5324..0000000 --- a/day2/second-crypto.txt +++ /dev/null @@ -1,51 +0,0 @@ -# Crypto - -## Overview: -Why do we need crypto? - -- Confidentiality -- Key Ex -- Identity -- Authentication - -## Asymetric Cryto -- Symmetric crypto faster, asymmetric slower and computational expensive -- RSA - - slightly old - - Needs 2048+ size keys - -## eliptic curve crypto -- Dot function -- Key exchange with shared key created from a -> aG -> aGb <- bG <- b -- Week to man in the middle - -## Web of trust -- How can we safely pass the key over an unsecured channel -- Just blindly trust on first use (like ssh) - -## The trusted third party -- Get a 3rd actor to meet Alice and Bob to add trust - -## hash functions -- md5 weak -- sha good - -# Merkel tree -- Hash chunks -- Combine chunks and hash -- Can work out which parts are wrong -- good for static data -- hash tree - -## BLock chain -- Use block chain to share trusted messages - -@giveupalready -https://github.com/carnage - - - - - --------- -Web of trust, trust once, trust forever diff --git a/links-and-useful-keywords.txt b/links-and-useful-keywords.txt new file mode 100644 index 0000000..8ff012b --- /dev/null +++ b/links-and-useful-keywords.txt @@ -0,0 +1,30 @@ +couch base +php_swoole +supervisorD +new relic +bugsnag +amazon cloud watch +sonarqube +sensolabs security checker +fastcgi_finish_request +crowdstar https://github.com/Crowdstar/background-processing +https://joind.in/event/php-uk-conference-2019/massively-scaled-high-performance-web-services-with-php +https://www.slideshare.net/DeminYin/massively-scaled-high-performance-web-services-with-php-132696547 +https://redis.io/commands/sadd +https://www.postgresql.org/docs/9.1/hstore.html +https://wiki.openstreetmap.org/wiki/PostgreSQL +https://php-ml.readthedocs.io/en/latest/machine-learning/workflow/pipeline/ +https://en.wikipedia.org/wiki/Benford%27s_law +https://liam-wiltshire.github.io/talks/?talk=machinelearning&conference=phpuk +https://joind.in/event/php-uk-conference-2019/learning-the-hows-and-whys-of-machine-learning +https://github.com/FriendsOfSymfony/FOSRestBundle +https://github.com/whiteoctober/Pagerfanta +https://www.ics.uci.edu/~fielding/pubs/dissertation/rest_arch_style.htm +https://en.wikipedia.org/wiki/Data_transfer_object +https://fractal.thephpleague.com/ +https://api-platform.com/ +https://www.sonatype.com/ +https://traefik.io/ +https://talks.bitexpert.de/phpuk19-gitlabci/ +https://docs.google.com/presentation/d/1Ogejf47b7k0RWU-7lE_uBsCmJxL3CvydOl8fVukuQ_4/edit +https://github.com/carnage -- cgit v1.2.3