Work/LancacheMinimal
1
0
Fork 0
Code Issues Activity

Initial commit

Browse Source
This commit is contained in:
Bram Prieshof
2024-08-12 23:08:39 +02:00
commit c75f447ab7
15 changed files with 624 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
.env Normal file
View File

@@ -0,0 +1,46 @@
## See the "Settings" section in README.md for more details
CACHE_DOMAINS_REPO="http://172.20.0.4/cache-domains.git"
CACHE_DOMAINS_BRANCH="master"
## Set this to true if you're using a load balancer, or set it to false if you're using separate IPs for each service.
## If you're using monolithic (the default), leave this set to true
USE_GENERIC_CACHE=true
## IP addresses that the lancache monolithic instance is reachable on
## Specify one or more IPs, space separated - these will be used when resolving DNS hostnames through lancachenet-dns. Multiple IPs can improve cache priming performance for some services (e.g. Steam)
## Note: This setting only affects DNS, monolithic and sniproxy will still bind to all IPs by default
LANCACHE_IP=192.168.0.103
## IP address on the host that the DNS server should bind to
DNS_BIND_IP=192.168.0.103
## DNS Resolution for forwarded DNS lookups
UPSTREAM_DNS=9.9.9.9
## Storage path for the cached data
## Note that by default, this will be a folder relative to the docker-compose.yml file
CACHE_ROOT=./lancache
## Change this to customise the maximum size of the disk cache (default 2000g).
## If you have more storage, you'll likely want to increase this.
## The cache server will prune content on a least-recently-used basis if it
## starts approaching this limit.
CACHE_DISK_SIZE=200g
## Sets the minimum free disk space that must be kept at all times.
## When the available free space drops below the set amount for any reason,
## the cache server will begin pruning content to free up space.
## Prevents accidentally running out of disk space if CACHE_DISK_SIZE is set too high.
MIN_FREE_DISK=10g
## Change this to allow sufficient index memory for the nginx cache manager (default 500m)
## We recommend 250m of index memory per 1TB of CACHE_DISK_SIZE
CACHE_INDEX_SIZE=500m
## Change this to limit the maximum age of cached content (default 3650d)
CACHE_MAX_AGE=3650d
## Set the timezone for the docker containers, useful for correct timestamps on logs (default Europe/London)
## Formatted as tz database names. Example: Europe/Oslo or America/Los_Angeles
TZ=Europe/Amsterdam

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
scripts/output
scripts/config.json

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 UK LAN Techs
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

25
README.md Executable file
View File

@@ -0,0 +1,25 @@
# Minimal cachelist for windows and apple updates
## License
The MIT License (MIT)
Copyright (c) 2017 UK LAN Techs
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

23
SETUP.md Normal file
View File

@@ -0,0 +1,23 @@
## Requirements
* Docker
* Docker-compose
```
apt install -y docker docker-compose
```
## Setup
Source: [Official Documentation](https://lancache.net/docs/)
```
git clone https://github.com/lancachenet/docker-compose/ lancache
cd lancache
#Update configuration (see below for more details)
nano .env
docker-compose up -d
```
Add the netxt lines to the top of the `.env` file
```
CACHE_DOMAINS_REPO="https://git.bprieshof.nl/Work/LancacheMinimal.git"
CACHE_DOMAINS_BRANCH="main"
```

5
apple.txt Normal file
View File

@@ -0,0 +1,5 @@
swcdn.apple.com
swdownload.apple.com
swquery.apple.com
swscan.apple.com
osrecovery.apple.com

14
cache_domains.json Normal file
View File

@@ -0,0 +1,14 @@
{
"cache_domains": [
{
"name": "wsus",
"description": "CDN for windows updates",
"domain_files": ["windowsupdates.txt"]
},
{
"name": "apple",
"description": "CDN for apple updates",
"domain_files": ["apple.txt"]
}
]
}

48
scripts/README.md Executable file
View File

@@ -0,0 +1,48 @@
# DNS Generation Scripts
## Introduction
The respective shell scripts contained within this directory can be utilised to generate application specific compliant
configuration which can be utilised with:
* Dnsmasq
* Unbound
* AdGuard Home
## Usage
1. Copy `config.example.json` to `config.json`.
2. Modify `config.json` to include your Cacheserver's IP(s) and the CDNs you plan to cache.
The following example assumes a single shared Cacheserver IP:
```json
{
"ips": {
"generic": ["10.10.10.200"]
},
"cache_domains": {
"blizzard": "generic",
"epicgames": "generic",
"nintendo": "generic",
"origin": "generic",
"riot": "generic",
"sony": "generic",
"steam": "generic",
"uplay": "generic",
"wsus": "generic"
}
}
```
3. Run generation script relative to your DNS implementation: `bash create-dnsmasq.sh`.
4. Copy files from `output/{dnsmasq,unbound}/*` to the respective locations for Dnsmasq/Unbound.
5. Restart Dnsmasq or Unbound.
### Notes for Dnsmasq users
**This also applies to users utilising the script alongside Pi-hole.**
Multi-IP Lancache setups are only supported with Dnsmasq or Pi-hole versions >= 2.86 or 2021.09 respectively.
### Notes for AdGuard Home users
1. In the `config.json`, you may want to add an entry for your non-cached DNS upstreams. You can input this in `ip.adguardhome_upstream` as an array.
2. Once you have ran the script, you can point the upstream list to the text file generated. For example: `upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"`

20
scripts/config.example.json Normal file
View File

@@ -0,0 +1,20 @@
{
"ips": {
"adguardhome_upstream": ["94.140.14.140", "tls://dns.google", "https://dns.google/dns-query"],
"steam": ["10.10.3.10", "10.10.3.11"],
"origin": "10.10.3.12",
"blizzard": "10.10.3.13",
"windows": "10.10.3.14",
"riot": "10.10.3.15",
"generic": "10.10.3.16"
},
"cache_domains": {
"default": "generic",
"blizzard": "blizzard",
"origin": "origin",
"riot": "riot",
"steam": "steam",
"wsus": "windows",
"xboxlive": "windows"
}
}

77
scripts/create-adguardhome.sh Normal file
View File

@@ -0,0 +1,77 @@
#!/bin/bash
basedir=".."
outputdir="output/adguardhome"
path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
while read -r line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip${line}"="${ip}"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
agh_upstreams=$(jq -r ".ips[\"adguardhome_upstream\"] | .[]" config.json)
while read -r line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename${line}"="${name}"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
# add upstreams
echo "${agh_upstreams}" >> "${outputdir}/cache-domains.txt"
while read -r entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
fi
if [[ ${!cachename} == "disabled" ]]; then
continue;
fi
cacheipname="cacheip${!cachename}"
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read -r fileid; do
while read -r filename; do
destfilename="cache-domains.txt" #$(echo $filename | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch ${outputfile}
while read -r fileentry; do
# Ignore comments, newlines and wildcards
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
for i in ${cacheip}; do
if grep -qx "\[/${parsed}/\]${i}" "${outputfile}"; then
continue
fi
echo "[/${parsed}/]${i}" >> "${outputfile}"
done
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
cat << EOF
Configuration generation completed.
Please point the setting upstream_dns_file in AdGuardHome.yaml to the generated file.
For example:
upstream_dns_file: "/root/cache-domains/scripts/output/adguardhome/cache-domains.txt"
EOF

72
scripts/create-dnsmasq.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/bin/bash
basedir=".."
outputdir="output/dnsmasq"
path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
while read -r line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip${line}"="${ip}"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
while read -r line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename${line}"="${name}"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
while read -r entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
fi
if [[ ${!cachename} == "disabled" ]]; then
continue;
fi
cacheipname="cacheip${!cachename}"
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read -r fileid; do
while read -r filename; do
destfilename=$(echo $filename | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch ${outputfile}
while read -r fileentry; do
# Ignore comments, newlines and wildcards
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
parsed=$(echo ${fileentry} | sed -e "s/^\*\.//")
for i in ${cacheip}; do
if ! grep -qx "address=/${parsed}/${i}" "${outputfile}"; then
echo "address=/${parsed}/${i}" >> "${outputfile}"
fi
if ! grep -qx "local=/${parsed}/" "${outputfile}"; then
echo "local=/${parsed}/" >> "${outputfile}"
fi
done
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
cat << EOF
Configuration generation completed.
Please copy the following files:
- ./${outputdir}/*.conf to /etc/dnsmasq/dnsmasq.d/
EOF

114
scripts/create-rpz.sh Executable file
View File

@@ -0,0 +1,114 @@
#!/bin/bash
basedir=".."
outputdir="output/rpz"
path="${basedir}/cache_domains.json"
basedomain=${1:-lancache.net}
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
while read line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip$line"="$ip"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
while read line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename$line"="$name"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
outputfile=${outputdir}/db.rpz.$basedomain
cat > $outputfile << EOF
\$TTL 60 ; default TTL
\$ORIGIN rpz.$basedomain.
@ SOA ns1.$basedomain. admin.$basedomain. (
$(date +%Y%m%d01) ; serial
604800 ; refresh (1 week)
600 ; retry (10 mins)
600 ; expire (10 mins)
600 ; minimum (10 mins)
)
NS ns1.$basedomain.
NS ns2.$basedomain.
EOF
while read entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
fi
if [[ ${!cachename} == "disabled" ]]; then
continue;
fi
cacheipname="cacheip${!cachename}"
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read fileid; do
while read filename; do
echo "" >> $outputfile
echo "; $(echo $filename | sed -e 's/.txt$//')" >> $outputfile
destfilename=$(echo $filename | sed -e 's/txt/conf/')
while read fileentry; do
# Ignore comments and newlines
if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
continue
fi
parsed=$(echo $fileentry)
if grep -qx "^\"${parsed}\". " $outputfile; then
continue
fi
t=""
for i in ${cacheip}; do
# only one cname per domain is allowed
if [[ ${t} = "CNAME" ]]; then
continue
fi
# for cnames you must use a fqdn with trailing dot
t="CNAME"
if [[ ${i} =~ ^[0-9\.]+$ ]] ; then
t="A"
elif [[ ! ${i} =~ \.$ ]] ; then
i="${i}."
fi
printf "%-50s IN %s %s\n" \
"${parsed}" \
"${t}" \
"${i}" \
>> $outputfile
done
done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
cat << EOF
Configuration generation completed.
Please include the rpz zone in your bind configuration"
- cp $outputfile /etc/bind
- configure the zone and use it
options {
[...]
response-policy {zone "rpz.$basedomain";};
[...]
}
zone "rpz.$basedomain" {
type master;
file "/etc/bind/db.rpz.$basedomain";
};
EOF

72
scripts/create-squid.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/bin/bash
basedir=".."
outputdir="output/squid"
path="${basedir}/cache_domains.json"
REGEX="^\\*\\.(.*)$"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
while read -r line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename${line}"="${name}"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
while read -r entry; do
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
fi
if [[ ${!cachename} == "disabled" ]]; then
continue;
fi
while read -r fileid; do
while read -r filename; do
destfilename=$(echo ${!cachename}.txt)
outputfile=${outputdir}/${destfilename}
touch ${outputfile}
while read -r fileentry; do
# Ignore comments
if [[ ${fileentry} == \#* ]] || [[ -z ${fileentry} ]]; then
continue
fi
# Handle wildcards to squid wildcards
parsed=$(echo ${fileentry} | sed -e "s/^\*\./\./")
# If we have cdn.thing and *.cdn.thing in cache_domains
# Squid requires ONLY cdn.thing
#
# If the fileentry starts with *.cdn.thing
if [[ ${fileentry} =~ $REGEX ]]; then
# Does the cache_domains file also contain cdn.thing
grep "${BASH_REMATCH[1]}" ${basedir}/${filename} | grep -v "${fileentry}" > /dev/null
if [[ $? -eq 0 ]]; then
# Skip *.cdn.thing as cdn.thing will be collected earlier/later
continue
fi
fi
echo "${parsed}" >> "${outputfile}"
done <<< $(cat ${basedir}/${filename} | sort);
done <<< $(jq -r ".cache_domains[${entry}].domain_files[$fileid]" ${path})
done <<< $(jq -r ".cache_domains[${entry}].domain_files | to_entries[] | .key" ${path})
done <<< $(jq -r '.cache_domains | to_entries[] | .key' ${path})
cat << EOF
Configuration generation completed.
Please copy the following files:
- ./${outputdir}/*.txt to /etc/squid/domains/
EOF

74
scripts/create-unbound.sh Executable file
View File

@@ -0,0 +1,74 @@
#!/bin/bash
basedir=".."
outputdir="output/unbound"
path="${basedir}/cache_domains.json"
export IFS=' '
test=$(which jq);
out=$?
if [ $out -gt 0 ] ; then
echo "This script requires jq to be installed."
echo "Your package manager should be able to find it"
exit 1
fi
cachenamedefault="disabled"
while read line; do
ip=$(jq ".ips[\"${line}\"]" config.json)
declare "cacheip$line"="$ip"
done <<< $(jq -r '.ips | to_entries[] | .key' config.json)
while read line; do
name=$(jq -r ".cache_domains[\"${line}\"]" config.json)
declare "cachename$line"="$name"
done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json)
rm -rf ${outputdir}
mkdir -p ${outputdir}
while read entry; do
unset cacheip
unset cachename
key=$(jq -r ".cache_domains[$entry].name" $path)
cachename="cachename${key}"
if [ -z "${!cachename}" ]; then
cachename="cachenamedefault"
fi
if [[ ${!cachename} == "disabled" ]]; then
continue;
fi
cacheipname="cacheip${!cachename}"
cacheip=$(jq -r 'if type == "array" then .[] else . end' <<< ${!cacheipname} | xargs)
while read fileid; do
while read filename; do
destfilename=$(echo $filename | sed -e 's/txt/conf/')
outputfile=${outputdir}/${destfilename}
touch $outputfile
while read fileentry; do
# Ignore comments and newlines
if [[ $fileentry == \#* ]] || [[ -z $fileentry ]]; then
continue
fi
parsed=$(echo $fileentry | sed -e "s/^\*\.//")
if grep -qx " local-zone: \"${parsed}\" redirect" $outputfile; then
continue
fi
if [[ $(head -n 1 $outputfile) != "server:" ]]; then
echo "server:" >> $outputfile
fi
echo " local-zone: \"${parsed}\" redirect" >> $outputfile
for i in ${cacheip}; do
echo " local-data: \"${parsed} 30 IN A ${i}\"" >> $outputfile
done
done <<< $(cat ${basedir}/$filename | sort);
done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path)
done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path)
done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path)
cat << EOF
Configuration generation completed.
Please copy the following files:
- ./${outputdir}/*.conf to /etc/unbound/unbound.conf.d/
EOF

11
windowsupdates.txt Normal file
View File

@@ -0,0 +1,11 @@
*.windowsupdate.com
*.dl.delivery.mp.microsoft.com
dl.delivery.mp.microsoft.com
*.update.microsoft.com
*.do.dsp.mp.microsoft.com
*.microsoft.com.edgesuite.net
amupdatedl.microsoft.com
amupdatedl2.microsoft.com
amupdatedl3.microsoft.com
amupdatedl4.microsoft.com
amupdatedl5.microsoft.com