File: /var/www/api-storage/node_modules/minio/src/internal/helper.ts
/*
* MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as crypto from 'node:crypto'
import * as stream from 'node:stream'
import { XMLParser } from 'fast-xml-parser'
import ipaddr from 'ipaddr.js'
import _ from 'lodash'
import * as mime from 'mime-types'
import { fsp, fstat } from './async.ts'
import type { Binary, Encryption, ObjectMetaData, RequestHeaders, ResponseHeader } from './type.ts'
import { ENCRYPTION_TYPES } from './type.ts'
const MetaDataHeaderPrefix = 'x-amz-meta-'
export function hashBinary(buf: Buffer, enableSHA256: boolean) {
let sha256sum = ''
if (enableSHA256) {
sha256sum = crypto.createHash('sha256').update(buf).digest('hex')
}
const md5sum = crypto.createHash('md5').update(buf).digest('base64')
return { md5sum, sha256sum }
}
// S3 percent-encodes some extra non-standard characters in a URI . So comply with S3.
const encodeAsHex = (c: string) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`
export function uriEscape(uriStr: string): string {
return encodeURIComponent(uriStr).replace(/[!'()*]/g, encodeAsHex)
}
export function uriResourceEscape(string: string) {
return uriEscape(string).replace(/%2F/g, '/')
}
export function getScope(region: string, date: Date, serviceName = 's3') {
return `${makeDateShort(date)}/${region}/${serviceName}/aws4_request`
}
/**
* isAmazonEndpoint - true if endpoint is 's3.amazonaws.com' or 's3.cn-north-1.amazonaws.com.cn'
*/
export function isAmazonEndpoint(endpoint: string) {
return endpoint === 's3.amazonaws.com' || endpoint === 's3.cn-north-1.amazonaws.com.cn'
}
/**
* isVirtualHostStyle - verify if bucket name is support with virtual
* hosts. bucketNames with periods should be always treated as path
* style if the protocol is 'https:', this is due to SSL wildcard
* limitation. For all other buckets and Amazon S3 endpoint we will
* default to virtual host style.
*/
export function isVirtualHostStyle(endpoint: string, protocol: string, bucket: string, pathStyle: boolean) {
if (protocol === 'https:' && bucket.includes('.')) {
return false
}
return isAmazonEndpoint(endpoint) || !pathStyle
}
export function isValidIP(ip: string) {
return ipaddr.isValid(ip)
}
/**
* @returns if endpoint is valid domain.
*/
export function isValidEndpoint(endpoint: string) {
return isValidDomain(endpoint) || isValidIP(endpoint)
}
/**
* @returns if input host is a valid domain.
*/
export function isValidDomain(host: string) {
if (!isString(host)) {
return false
}
// See RFC 1035, RFC 3696.
if (host.length === 0 || host.length > 255) {
return false
}
// Host cannot start or end with a '-'
if (host[0] === '-' || host.slice(-1) === '-') {
return false
}
// Host cannot start or end with a '_'
if (host[0] === '_' || host.slice(-1) === '_') {
return false
}
// Host cannot start with a '.'
if (host[0] === '.') {
return false
}
const nonAlphaNumerics = '`~!@#$%^&*()+={}[]|\\"\';:><?/'
// All non alphanumeric characters are invalid.
for (const char of nonAlphaNumerics) {
if (host.includes(char)) {
return false
}
}
// No need to regexp match, since the list is non-exhaustive.
// We let it be valid and fail later.
return true
}
/**
* Probes contentType using file extensions.
*
* @example
* ```
* // return 'image/png'
* probeContentType('file.png')
* ```
*/
export function probeContentType(path: string) {
let contentType = mime.lookup(path)
if (!contentType) {
contentType = 'application/octet-stream'
}
return contentType
}
/**
* is input port valid.
*/
export function isValidPort(port: unknown): port is number {
// Convert string port to number if needed
const portNum = typeof port === 'string' ? parseInt(port, 10) : port
// verify if port is a valid number
if (!isNumber(portNum) || isNaN(portNum)) {
return false
}
// port `0` is valid and special case
return 0 <= portNum && portNum <= 65535
}
export function isValidBucketName(bucket: unknown) {
if (!isString(bucket)) {
return false
}
// bucket length should be less than and no more than 63
// characters long.
if (bucket.length < 3 || bucket.length > 63) {
return false
}
// bucket with successive periods is invalid.
if (bucket.includes('..')) {
return false
}
// bucket cannot have ip address style.
if (/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/.test(bucket)) {
return false
}
// bucket should begin with alphabet/number and end with alphabet/number,
// with alphabet/number/.- in the middle.
if (/^[a-z0-9][a-z0-9.-]+[a-z0-9]$/.test(bucket)) {
return true
}
return false
}
/**
* check if objectName is a valid object name
*/
export function isValidObjectName(objectName: unknown) {
if (!isValidPrefix(objectName)) {
return false
}
return objectName.length !== 0
}
/**
* check if prefix is valid
*/
export function isValidPrefix(prefix: unknown): prefix is string {
if (!isString(prefix)) {
return false
}
if (prefix.length > 1024) {
return false
}
return true
}
/**
* check if typeof arg number
*/
export function isNumber(arg: unknown): arg is number {
return typeof arg === 'number'
}
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export type AnyFunction = (...args: any[]) => any
/**
* check if typeof arg function
*/
export function isFunction(arg: unknown): arg is AnyFunction {
return typeof arg === 'function'
}
/**
* check if typeof arg string
*/
export function isString(arg: unknown): arg is string {
return typeof arg === 'string'
}
/**
* check if typeof arg object
*/
export function isObject(arg: unknown): arg is object {
return typeof arg === 'object' && arg !== null
}
/**
* check if object is readable stream
*/
export function isReadableStream(arg: unknown): arg is stream.Readable {
// eslint-disable-next-line @typescript-eslint/unbound-method
return isObject(arg) && isFunction((arg as stream.Readable)._read)
}
/**
* check if arg is boolean
*/
export function isBoolean(arg: unknown): arg is boolean {
return typeof arg === 'boolean'
}
export function isEmpty(o: unknown): o is null | undefined {
return _.isEmpty(o)
}
export function isEmptyObject(o: Record<string, unknown>): boolean {
return Object.values(o).filter((x) => x !== undefined).length !== 0
}
export function isDefined<T>(o: T): o is Exclude<T, null | undefined> {
return o !== null && o !== undefined
}
/**
* check if arg is a valid date
*/
export function isValidDate(arg: unknown): arg is Date {
// @ts-expect-error checknew Date(Math.NaN)
return arg instanceof Date && !isNaN(arg)
}
/**
* Create a Date string with format: 'YYYYMMDDTHHmmss' + Z
*/
export function makeDateLong(date?: Date): string {
date = date || new Date()
// Gives format like: '2017-08-07T16:28:59.889Z'
const s = date.toISOString()
return s.slice(0, 4) + s.slice(5, 7) + s.slice(8, 13) + s.slice(14, 16) + s.slice(17, 19) + 'Z'
}
/**
* Create a Date string with format: 'YYYYMMDD'
*/
export function makeDateShort(date?: Date) {
date = date || new Date()
// Gives format like: '2017-08-07T16:28:59.889Z'
const s = date.toISOString()
return s.slice(0, 4) + s.slice(5, 7) + s.slice(8, 10)
}
/**
* pipesetup sets up pipe() from left to right os streams array
* pipesetup will also make sure that error emitted at any of the upstream Stream
* will be emitted at the last stream. This makes error handling simple
*/
export function pipesetup(...streams: [stream.Readable, ...stream.Duplex[], stream.Writable]) {
// @ts-expect-error ts can't narrow this
return streams.reduce((src: stream.Readable, dst: stream.Writable) => {
src.on('error', (err) => dst.emit('error', err))
return src.pipe(dst)
})
}
/**
* return a Readable stream that emits data
*/
export function readableStream(data: unknown): stream.Readable {
const s = new stream.Readable()
s._read = () => {}
s.push(data)
s.push(null)
return s
}
/**
* Process metadata to insert appropriate value to `content-type` attribute
*/
export function insertContentType(metaData: ObjectMetaData, filePath: string): ObjectMetaData {
// check if content-type attribute present in metaData
for (const key in metaData) {
if (key.toLowerCase() === 'content-type') {
return metaData
}
}
// if `content-type` attribute is not present in metadata, then infer it from the extension in filePath
return {
...metaData,
'content-type': probeContentType(filePath),
}
}
/**
* Function prepends metadata with the appropriate prefix if it is not already on
*/
export function prependXAMZMeta(metaData?: ObjectMetaData): RequestHeaders {
if (!metaData) {
return {}
}
return _.mapKeys(metaData, (value, key) => {
if (isAmzHeader(key) || isSupportedHeader(key) || isStorageClassHeader(key)) {
return key
}
return MetaDataHeaderPrefix + key
})
}
/**
* Checks if it is a valid header according to the AmazonS3 API
*/
export function isAmzHeader(key: string) {
const temp = key.toLowerCase()
return (
temp.startsWith(MetaDataHeaderPrefix) ||
temp === 'x-amz-acl' ||
temp.startsWith('x-amz-server-side-encryption-') ||
temp === 'x-amz-server-side-encryption'
)
}
/**
* Checks if it is a supported Header
*/
export function isSupportedHeader(key: string) {
const supported_headers = [
'content-type',
'cache-control',
'content-encoding',
'content-disposition',
'content-language',
'x-amz-website-redirect-location',
'if-none-match',
'if-match',
]
return supported_headers.includes(key.toLowerCase())
}
/**
* Checks if it is a storage header
*/
export function isStorageClassHeader(key: string) {
return key.toLowerCase() === 'x-amz-storage-class'
}
export function extractMetadata(headers: ResponseHeader) {
return _.mapKeys(
_.pickBy(headers, (value, key) => isSupportedHeader(key) || isStorageClassHeader(key) || isAmzHeader(key)),
(value, key) => {
const lower = key.toLowerCase()
if (lower.startsWith(MetaDataHeaderPrefix)) {
return lower.slice(MetaDataHeaderPrefix.length)
}
return key
},
)
}
export function getVersionId(headers: ResponseHeader = {}) {
return headers['x-amz-version-id'] || null
}
export function getSourceVersionId(headers: ResponseHeader = {}) {
return headers['x-amz-copy-source-version-id'] || null
}
export function sanitizeETag(etag = ''): string {
const replaceChars: Record<string, string> = {
'"': '',
'"': '',
'"': '',
'"': '',
'"': '',
}
return etag.replace(/^("|"|")|("|"|")$/g, (m) => replaceChars[m] as string)
}
export function toMd5(payload: Binary): string {
// use string from browser and buffer from nodejs
// browser support is tested only against minio server
return crypto.createHash('md5').update(Buffer.from(payload)).digest().toString('base64')
}
export function toSha256(payload: Binary): string {
return crypto.createHash('sha256').update(payload).digest('hex')
}
/**
* toArray returns a single element array with param being the element,
* if param is just a string, and returns 'param' back if it is an array
* So, it makes sure param is always an array
*/
export function toArray<T = unknown>(param: T | T[]): Array<T> {
if (!Array.isArray(param)) {
return [param] as T[]
}
return param
}
export function sanitizeObjectKey(objectName: string): string {
// + symbol characters are not decoded as spaces in JS. so replace them first and decode to get the correct result.
const asStrName = (objectName ? objectName.toString() : '').replace(/\+/g, ' ')
return decodeURIComponent(asStrName)
}
export function sanitizeSize(size?: string): number | undefined {
return size ? Number.parseInt(size) : undefined
}
export const PART_CONSTRAINTS = {
// absMinPartSize - absolute minimum part size (5 MiB)
ABS_MIN_PART_SIZE: 1024 * 1024 * 5,
// MIN_PART_SIZE - minimum part size 16MiB per object after which
MIN_PART_SIZE: 1024 * 1024 * 16,
// MAX_PARTS_COUNT - maximum number of parts for a single multipart session.
MAX_PARTS_COUNT: 10000,
// MAX_PART_SIZE - maximum part size 5GiB for a single multipart upload
// operation.
MAX_PART_SIZE: 1024 * 1024 * 1024 * 5,
// MAX_SINGLE_PUT_OBJECT_SIZE - maximum size 5GiB of object per PUT
// operation.
MAX_SINGLE_PUT_OBJECT_SIZE: 1024 * 1024 * 1024 * 5,
// MAX_MULTIPART_PUT_OBJECT_SIZE - maximum size 5TiB of object for
// Multipart operation.
MAX_MULTIPART_PUT_OBJECT_SIZE: 1024 * 1024 * 1024 * 1024 * 5,
}
const GENERIC_SSE_HEADER = 'X-Amz-Server-Side-Encryption'
const ENCRYPTION_HEADERS = {
// sseGenericHeader is the AWS SSE header used for SSE-S3 and SSE-KMS.
sseGenericHeader: GENERIC_SSE_HEADER,
// sseKmsKeyID is the AWS SSE-KMS key id.
sseKmsKeyID: GENERIC_SSE_HEADER + '-Aws-Kms-Key-Id',
} as const
/**
* Return Encryption headers
* @param encConfig
* @returns an object with key value pairs that can be used in headers.
*/
export function getEncryptionHeaders(encConfig: Encryption): RequestHeaders {
const encType = encConfig.type
if (!isEmpty(encType)) {
if (encType === ENCRYPTION_TYPES.SSEC) {
return {
[ENCRYPTION_HEADERS.sseGenericHeader]: 'AES256',
}
} else if (encType === ENCRYPTION_TYPES.KMS) {
return {
[ENCRYPTION_HEADERS.sseGenericHeader]: encConfig.SSEAlgorithm,
[ENCRYPTION_HEADERS.sseKmsKeyID]: encConfig.KMSMasterKeyID,
}
}
}
return {}
}
export function partsRequired(size: number): number {
const maxPartSize = PART_CONSTRAINTS.MAX_MULTIPART_PUT_OBJECT_SIZE / (PART_CONSTRAINTS.MAX_PARTS_COUNT - 1)
let requiredPartSize = size / maxPartSize
if (size % maxPartSize > 0) {
requiredPartSize++
}
requiredPartSize = Math.trunc(requiredPartSize)
return requiredPartSize
}
/**
* calculateEvenSplits - computes splits for a source and returns
* start and end index slices. Splits happen evenly to be sure that no
* part is less than 5MiB, as that could fail the multipart request if
* it is not the last part.
*/
export function calculateEvenSplits<T extends { Start?: number }>(
size: number,
objInfo: T,
): {
startIndex: number[]
objInfo: T
endIndex: number[]
} | null {
if (size === 0) {
return null
}
const reqParts = partsRequired(size)
const startIndexParts: number[] = []
const endIndexParts: number[] = []
let start = objInfo.Start
if (isEmpty(start) || start === -1) {
start = 0
}
const divisorValue = Math.trunc(size / reqParts)
const reminderValue = size % reqParts
let nextStart = start
for (let i = 0; i < reqParts; i++) {
let curPartSize = divisorValue
if (i < reminderValue) {
curPartSize++
}
const currentStart = nextStart
const currentEnd = currentStart + curPartSize - 1
nextStart = currentEnd + 1
startIndexParts.push(currentStart)
endIndexParts.push(currentEnd)
}
return { startIndex: startIndexParts, endIndex: endIndexParts, objInfo: objInfo }
}
const fxp = new XMLParser({ numberParseOptions: { eNotation: false, hex: true, leadingZeros: true } })
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function parseXml(xml: string): any {
const result = fxp.parse(xml)
if (result.Error) {
throw result.Error
}
return result
}
/**
* get content size of object content to upload
*/
export async function getContentLength(s: stream.Readable | Buffer | string): Promise<number | null> {
// use length property of string | Buffer
if (typeof s === 'string' || Buffer.isBuffer(s)) {
return s.length
}
// property of `fs.ReadStream`
const filePath = (s as unknown as Record<string, unknown>).path as string | undefined
if (filePath && typeof filePath === 'string') {
const stat = await fsp.lstat(filePath)
return stat.size
}
// property of `fs.ReadStream`
const fd = (s as unknown as Record<string, unknown>).fd as number | null | undefined
if (fd && typeof fd === 'number') {
const stat = await fstat(fd)
return stat.size
}
return null
}